Inspect your data !

Your example works perfect as shown next.

create table #mytb1 (col1 nvarchar(50) not null);

insert into #mytb1

Select '500910000000000000'

union all Select '500903000000000000'

union all Select '500909000000000000'

union all Select '500903000000000000'

union all Select '500805000000000000'

union all Select '500911000000000000'

union all Select '500905000000000000'

union all Select '500911000000000000'

union all Select '500905000000000000'

union all Select '500911000000000000';

-- (10 row(s) affected)

Alter table #mytb1

add colnew bigint null ;

go

update #mytb1

set colnew = cast( col1 as bigint )

where colnew is null ;

-- (10 row(s) affected)

drop table #mytb1;