make-manuf.py: Add response headers.

Add comments containing the resonse headers for the URLs we fetch.
standards-oui.ieee.org currently returns inconsistent results depending
on which host you happen to resolve.

Change-Id: I4adba7e51628d0350ba8e091523807ec85009700
Reviewed-on: https://code.wireshark.org/review/29729
Reviewed-by: Gerald Combs <gerald@wireshark.org>

tools/make-manuf.py

@@ -47,19 +47,25 @@ def exit_msg(msg=None, status=1):
     sys.exit(status)
 
 def open_url(url):
+    '''Open a URL.
+    Returns a tuple containing the body and response dict. The body is a
+    str in Python 3 and bytes in Python 2 in order to be compatibile with
+    csv.reader.
+    '''
     req_headers = { 'User-Agent': 'Wireshark make-manuf' }
     try:
         if sys.version_info[0] >= 3:
             req = urllib.request.Request(url, headers=req_headers)
-            url_fo = urllib.request.urlopen(req)
-            url_fd = codecs.getreader('utf8')(url_fo)
+            response = urllib.request.urlopen(req)
+            body = response.read().decode('UTF-8', 'replace')
         else:
             req = urllib2.Request(url, headers=req_headers)
-            url_fd = urllib2.urlopen(req)
+            response = urllib2.urlopen(req)
+            body = response.read()
     except:
         exit_msg('Error opening ' + url)
 
-    return url_fd
+    return (body, dict(response.info()))
 
 def shorten(manuf):
     '''Convert a long manufacturer name to abbreviated and short names'''
@@ -182,13 +188,20 @@ def main():
         ieee_d[db]['added'] = 0
         ieee_d[db]['total'] = 0
         print('Merging {} data from {}'.format(db, db_url))
-        ieee_fd = open_url(db_url)
-        ieee_csv = csv.reader(ieee_fd)
-        #Registry,Assignment,Organization Name,Organization Address
-        #IAB,0050C2DD6,Transas Marine Limited,Datavagen 37 Askim Vastra Gotaland SE 436 32
+        (body, response_d) = open_url(db_url)
+        ieee_csv = csv.reader(body.splitlines())
+        if sys.version_info[0] >= 3:
+            ieee_d[db]['last-modified'] = response_d['Last-Modified']
+            ieee_d[db]['length'] = response_d['Content-Length']
+        else:
+            ieee_d[db]['last-modified'] = response_d['last-modified']
+            ieee_d[db]['length'] = response_d['content-length']
+
         # Pop the title row.
         next(ieee_csv)
         for ieee_row in ieee_csv:
+            #Registry,Assignment,Organization Name,Organization Address
+            #IAB,0050C2DD6,Transas Marine Limited,Datavagen 37 Askim Vastra Gotaland SE 436 32
             oui = prefix_to_oui(ieee_row[1].upper())
             if sys.version_info[0] >= 3:
                 manuf = ieee_row[2].strip()
@@ -220,6 +233,15 @@ def main():
     manuf_fd.write(u"# Don't change it directly, change manuf.tmpl instead.\n#\n")
     manuf_fd.write('\n'.join(header_l))
 
+    for db in ieee_db_l:
+        manuf_fd.write(
+            u'''\
+# {url}:
+#   Content-Length: {length}
+#   Last-Modified: {last-modified}
+
+'''.format( **ieee_d[db]))
+
     oui_l = list(oui_d.keys())
     oui_l.sort()
     for oui in oui_l: