test: Extend white space check to verify files are valid utf-8

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>
This commit is contained in:
Kevin O'Connor 2018-07-26 16:00:39 -04:00
parent 09a64d24f1
commit 7c7de85f01
1 changed files with 8 additions and 2 deletions

View File

@ -4,7 +4,7 @@
# Copyright (C) 2018 Kevin O'Connor <kevin@koconnor.net> # Copyright (C) 2018 Kevin O'Connor <kevin@koconnor.net>
# #
# This file may be distributed under the terms of the GNU GPLv3 license. # This file may be distributed under the terms of the GNU GPLv3 license.
import sys, os.path import sys, os.path, unicodedata
HaveError = False HaveError = False
@ -28,10 +28,16 @@ def check_file(filename):
# Do checks # Do checks
lineno = 0 lineno = 0
for lineno, line in enumerate(data.split('\n')): for lineno, line in enumerate(data.split('\n')):
# Verify line is valid utf-8
try:
line = line.decode('utf-8')
except UnicodeDecodeError:
report_error(filename, lineno, "Non utf-8 character")
continue
# Check for control characters # Check for control characters
for c in line: for c in line:
oc = ord(c) oc = ord(c)
if oc < 32: if oc < 32 or unicodedata.category(c).startswith('C'):
char_name = repr(c) char_name = repr(c)
if oc == 9: if oc == 9:
if os.path.basename(filename).lower() == 'makefile': if os.path.basename(filename).lower() == 'makefile':