update char counting to account for multiple unicode points

jeremylow · jeremylow · commit cd52ca74eaa3 · 2018-02-17T11:51:52.000-05:00
diff --git a/tests/test_twitter_utils.py b/tests/test_twitter_utils.py
@@ -1,4 +1,5 @@
 # encoding: utf-8
+from __future__ import unicode_literals
 
 import unittest
 
@@ -81,3 +82,11 @@ def test_calc_expected_status_length_with_url_and_extra_spaces(self):
         status = 'hi a tweet          there               example.com'
         len_status = calc_expected_status_length(status)
         self.assertEqual(len_status, 63)
+
+    def test_calc_expected_status_length_with_wide_unicode(self):
+        status = "&hellip;"
+        len_status = calc_expected_status_length(status)
+        assert len_status == 2
+        status = "&hellip;&hellip;"
+        len_status = calc_expected_status_length(status)
+        assert len_status == 4
diff --git a/twitter/api.py b/twitter/api.py
@@ -896,12 +896,12 @@ def GetStatuses(self,
             'map': map
         }
         while offset < len(status_ids):
-            parameters['id'] = ','.join([str(enf_type('status_id', int, status_id)) for status_id in status_ids[offset:offset+100]])
+            parameters['id'] = ','.join([str(enf_type('status_id', int, status_id)) for status_id in status_ids[offset:offset + 100]])
 
             resp = self._RequestUrl(url, 'GET', data=parameters)
             data = self._ParseAndCheckTwitter(resp.content.decode('utf-8'))
             if map:
-                result.update({int(key):(Status.NewFromJsonDict(value) if value else None) for key,value in data['id'].items()})
+                result.update({int(key): (Status.NewFromJsonDict(value) if value else None) for key, value in data['id'].items()})
             else:
                 result += [Status.NewFromJsonDict(dataitem) for dataitem in data]
 
diff --git a/twitter/twitter_utils.py b/twitter/twitter_utils.py
@@ -4,7 +4,9 @@
 import mimetypes
 import os
 import re
+import sys
 from tempfile import NamedTemporaryFile
+from unicodedata import normalize
 
 try:
     from urllib.parse import urlparse
@@ -14,6 +16,14 @@
 import requests
 from twitter import TwitterError
 
+if sys.version_info < (3,):
+    range = xrange
+
+CHAR_RANGES = [
+    range(0, 4351),
+    range(8192, 8205),
+    range(8208, 8223),
+    range(8242, 8247)]
 
 TLDS = [
     "ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar",
@@ -171,7 +181,11 @@ def calc_expected_status_length(status, short_url_length=23):
         if is_url(word):
             status_length += short_url_length
         else:
-            status_length += len(word)
+            for character in word:
+                if any([ord(normalize("NFC", character)) in char_range for char_range in CHAR_RANGES]):
+                    status_length += 1
+                else:
+                    status_length += 2
     status_length += len(re.findall(r'\s', status))
     return status_length
 
