| Message ID | 20191118164647.29409-3-ross.burton@intel.com |
|---|---|
| State | Accepted |
| Commit | 6b73004668b3b71c9c38814b79fbb58c893ed434 |
| Headers | show |
| Series | [1/6] cve-update-db-native: don't hardcode the database name | expand |
On 11/18/19 10:46 AM, Ross Burton wrote: > urllib handles adding proxy handlers if the proxies are set in the environment, > so call bb.utils.export_proxies() to do that and remove the manual setup. > > Signed-off-by: Ross Burton <ross.burton@intel.com> > --- > .../recipes-core/meta/cve-update-db-native.bb | 31 +++---------------- > 1 file changed, 5 insertions(+), 26 deletions(-) > > diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb > index 08b18f064f0..db1d69a28e5 100644 > --- a/meta/recipes-core/meta/cve-update-db-native.bb > +++ b/meta/recipes-core/meta/cve-update-db-native.bb > @@ -21,10 +21,12 @@ python do_populate_cve_db() { > """ > Update NVD database with json data feed > """ > - > + import bb.utils > import sqlite3, urllib, urllib.parse, shutil, gzip > from datetime import date > > + bb.utils.export_proxies(d) > + > BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" > YEAR_START = 2002 Two comments, I know unrelated to this specific commit, but I noticed them while looking... The current NVD data is now in the '1.1' format. I was lead to believe the 1.0 feeds would be stopped at some point. Second, if we're successful with some of the SRTool components, we should be able to export the data into NVD format. So in that case, it would be nice to be able to point the cve-update components to an alternative datasource. (I do assume the data format is the same.) --Mark > @@ -40,16 +42,6 @@ python do_populate_cve_db() { > except OSError: > pass > > - proxy = d.getVar("https_proxy") > - if proxy: > - # instantiate an opener but do not install it as the global > - # opener unless if we're really sure it's applicable for all > - # urllib requests > - proxy_handler = urllib.request.ProxyHandler({'https': proxy}) > - proxy_opener = urllib.request.build_opener(proxy_handler) > - else: > - proxy_opener = None > - > cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') > > if not os.path.isdir(db_dir): > @@ -67,15 +59,7 @@ python do_populate_cve_db() { > json_url = year_url + ".json.gz" > > # Retrieve meta last modified date > - > - response = None > - > - if proxy_opener: > - response = proxy_opener.open(meta_url) > - else: > - req = urllib.request.Request(meta_url) > - response = urllib.request.urlopen(req) > - > + response = urllib.request.urlopen(meta_url) > if response: > for l in response.read().decode("utf-8").splitlines(): > key, value = l.split(":", 1) > @@ -95,12 +79,7 @@ python do_populate_cve_db() { > > # Update db with current year json file > try: > - if proxy_opener: > - response = proxy_opener.open(json_url) > - else: > - req = urllib.request.Request(json_url) > - response = urllib.request.urlopen(req) > - > + response = urllib.request.urlopen(json_url) > if response: > update_db(c, gzip.decompress(response.read()).decode('utf-8')) > c.execute("insert or replace into META values (?, ?)", [year, last_modified]) > -- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb index 08b18f064f0..db1d69a28e5 100644 --- a/meta/recipes-core/meta/cve-update-db-native.bb +++ b/meta/recipes-core/meta/cve-update-db-native.bb @@ -21,10 +21,12 @@ python do_populate_cve_db() { """ Update NVD database with json data feed """ - + import bb.utils import sqlite3, urllib, urllib.parse, shutil, gzip from datetime import date + bb.utils.export_proxies(d) + BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-" YEAR_START = 2002 @@ -40,16 +42,6 @@ python do_populate_cve_db() { except OSError: pass - proxy = d.getVar("https_proxy") - if proxy: - # instantiate an opener but do not install it as the global - # opener unless if we're really sure it's applicable for all - # urllib requests - proxy_handler = urllib.request.ProxyHandler({'https': proxy}) - proxy_opener = urllib.request.build_opener(proxy_handler) - else: - proxy_opener = None - cve_f = open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') if not os.path.isdir(db_dir): @@ -67,15 +59,7 @@ python do_populate_cve_db() { json_url = year_url + ".json.gz" # Retrieve meta last modified date - - response = None - - if proxy_opener: - response = proxy_opener.open(meta_url) - else: - req = urllib.request.Request(meta_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(meta_url) if response: for l in response.read().decode("utf-8").splitlines(): key, value = l.split(":", 1) @@ -95,12 +79,7 @@ python do_populate_cve_db() { # Update db with current year json file try: - if proxy_opener: - response = proxy_opener.open(json_url) - else: - req = urllib.request.Request(json_url) - response = urllib.request.urlopen(req) - + response = urllib.request.urlopen(json_url) if response: update_db(c, gzip.decompress(response.read()).decode('utf-8')) c.execute("insert or replace into META values (?, ?)", [year, last_modified])
urllib handles adding proxy handlers if the proxies are set in the environment, so call bb.utils.export_proxies() to do that and remove the manual setup. Signed-off-by: Ross Burton <ross.burton@intel.com> --- .../recipes-core/meta/cve-update-db-native.bb | 31 +++---------------- 1 file changed, 5 insertions(+), 26 deletions(-) -- 2.20.1 -- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core