It's not really clear how LineNo joins, but this gives the correct result.
WITH Questions(Contact,[LineNo],LineNoMax,Question) AS (
SELECT a.Contact,a.[LineNo],MIN(b.[LineNo])-1,a.Question
FROM #Question a
LEFT OUTER JOIN #Question b ON b.Contact=a.Contact AND b.[LineNo]>a.[LineNo]
GROUP BY a.Contact,a.[LineNo],a.Question)
SELECT a.Contact,a.Question,b.Answer
FROM Questions a
INNER JOIN #Answer b ON b.Contact=a.Contact AND b.[LineNo] BETWEEN a.[LineNo] AND COALESCE(a.LineNoMax,b.[LineNo])