Maybe we're thinking a little bit different. Here's the solution I came with and a possible improvement using a temp table. If you're going to filter your information remember to do it during the load of the data into the CTE or temp table and not after.
--CTE Version
WITH AllDates AS(
SELECT starttime, vendorid, timeslotid, customerid
FROM dbo.book
UNION ALL
SELECT endtime, vendorid, timeslotid, NULL
FROM dbo.book
UNION ALL
SELECT FromTime, vendorid, schedid, NULL
FROM vendor_sched
UNION ALL
SELECT ToTime, vendorid, schedid, NULL
FROM vendor_sched
),
ContinuousIDs AS(
SELECT *, ROW_NUMBER() OVER( ORDER BY vendorid, timeslotid, starttime, customerid) row_id
FROM AllDates)
SELECT MIN( c1.starttime) OVER(PARTITION BY c1.vendorid, c1.timeslotid) AS SchedStartTime,
MAX( c2.starttime) OVER(PARTITION BY c1.vendorid, c1.timeslotid) AS SchedEndTime,
c1.VendorId,
c1.timeslotid AS SchedId,
c1.starttime AS StartTime,
c2.starttime AS EndTime,
c1.CustomerId
FROM ContinuousIDs c1
JOIN ContinuousIDs c2 ON c1.row_id +1 = c2.row_id AND c1.vendorid = c2.vendorid AND c1.timeslotid = c2.timeslotid
WHERE DATEDIFF( MINUTE, c1.starttime, c2.starttime) > 0
--Temp table version
CREATE TABLE #Temp(
rowid int IDENTITY,
starttimedatetime,
vendoridint,
timeslotidint,
customeridint,
CONSTRAINT PK_Temp PRIMARY KEY( rowid, vendorid, timeslotid))
INSERT INTO #Temp
SELECT starttime, vendorid, timeslotid, customerid
FROM dbo.book
UNION ALL
SELECT endtime, vendorid, timeslotid, NULL
FROM dbo.book
UNION ALL
SELECT FromTime, vendorid, schedid, NULL
FROM vendor_sched
UNION ALL
SELECT ToTime, vendorid, schedid, NULL
FROM vendor_sched
ORDER BY vendorid, timeslotid, starttime, customerid;
SELECT MIN( c1.starttime) OVER(PARTITION BY c1.vendorid, c1.timeslotid) AS SchedStartTime,
MAX( c2.starttime) OVER(PARTITION BY c1.vendorid, c1.timeslotid) AS SchedEndTime,
c1.VendorId,
c1.timeslotid AS SchedId,
c1.starttime AS StartTime,
c2.starttime AS EndTime,
c1.CustomerId
FROM #Temp c1
JOIN #Temp c2 ON c1.rowid +1 = c2.rowid AND c1.vendorid = c2.vendorid AND c1.timeslotid = c2.timeslotid
WHERE DATEDIFF( MINUTE, c1.starttime, c2.starttime) > 0
DROP TABLE #Temp
Remember to test and understand every part of the code before implementing it. If you have more questions, feel free to ask them.