From 7c7de85f01417152a8dd8956f7804d7651ac0398 Mon Sep 17 00:00:00 2001 From: Kevin O'Connor Date: Thu, 26 Jul 2018 16:00:39 -0400 Subject: [PATCH] test: Extend white space check to verify files are valid utf-8 Signed-off-by: Kevin O'Connor --- scripts/check_whitespace.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/check_whitespace.py b/scripts/check_whitespace.py index 6f32c24f..13c4a3ed 100755 --- a/scripts/check_whitespace.py +++ b/scripts/check_whitespace.py @@ -4,7 +4,7 @@ # Copyright (C) 2018 Kevin O'Connor # # This file may be distributed under the terms of the GNU GPLv3 license. -import sys, os.path +import sys, os.path, unicodedata HaveError = False @@ -28,10 +28,16 @@ def check_file(filename): # Do checks lineno = 0 for lineno, line in enumerate(data.split('\n')): + # Verify line is valid utf-8 + try: + line = line.decode('utf-8') + except UnicodeDecodeError: + report_error(filename, lineno, "Non utf-8 character") + continue # Check for control characters for c in line: oc = ord(c) - if oc < 32: + if oc < 32 or unicodedata.category(c).startswith('C'): char_name = repr(c) if oc == 9: if os.path.basename(filename).lower() == 'makefile':