Something like this:
CREATE TABLE Insurance (
    RecordNo INT IDENTITY,
    PolicyNo INT NOT NULL);
GO

INSERT INTO Insurance (PolicyNo)
VALUES (34564),(67548),(34564),(98271),(90198),(98271);

CTE to identify and delete duplicates (where ROW_NUMBER() > 1)
;WITH ctePolicyDupes(PolicyNo, dupeNumber)
AS
(SELECT i.PolicyNo
    , ROW_NUMBER() OVER (PARTITION BY i.PolicyNo ORDER BY RecordNo) dupeNo
FROM Insurance i)
DELETE FROM ctePolicyDupes
WHERE dupeNumber>1;