CRF reading code integrated into doit

code/UNFCCC_CRF_reader/UNFCCC_CRF_reader_core.py

@@ -16,8 +16,8 @@ from operator import itemgetter
 from collections import Counter
 from typing import Dict, List, Optional, Tuple, Union
 from datetime import datetime
-from . import crf_specifications as crf
-from .utils import downloaded_data_path
+import crf_specifications as crf
+from util import downloaded_data_path
 
 
 ### reading functions
@@ -931,7 +931,6 @@ def get_submission_dates(
                          f"the function's purpose is to return available dates.")
 
     if folder.exists():
-        print(folder)
         files = filter_filenames(folder.glob("*.xlsx"), **file_filter)
     else:
         raise ValueError(f"Folder {folder} does not exist")

code/UNFCCC_CRF_reader/UNFCCC_CRF_reader_prod.py

@@ -13,20 +13,21 @@ from datetime import date
 from pathlib import Path
 from typing import Optional
 
-from . import crf_specifications as crf
+#from . import crf_specifications as crf
+import crf_specifications as crf
 
-from .UNFCCC_CRF_reader_core import read_crf_table
-from .UNFCCC_CRF_reader_core import convert_crf_table_to_pm2if
-from .UNFCCC_CRF_reader_core import get_latest_date_for_country
-from .UNFCCC_CRF_reader_core import get_crf_files
-from .UNFCCC_CRF_reader_devel import save_unknown_categories_info
-from .UNFCCC_CRF_reader_devel import save_last_row_info
+from UNFCCC_CRF_reader_core import read_crf_table
+from UNFCCC_CRF_reader_core import convert_crf_table_to_pm2if
+from UNFCCC_CRF_reader_core import get_latest_date_for_country
+from UNFCCC_CRF_reader_core import get_crf_files
+from UNFCCC_CRF_reader_devel import save_unknown_categories_info
+from UNFCCC_CRF_reader_devel import save_last_row_info
 
-from .utils import code_path, log_path, \
+from util import code_path, log_path, \
     custom_country_mapping, extracted_data_path, root_path
 
 import sys
-sys.path.append('../UNFCCC_reader')
+sys.path.append(code_path.name)
 from UNFCCC_reader.get_submissions_info import get_country_code
 
 
@@ -187,7 +188,7 @@ def read_crf_for_country(
         pm2.pm2io.write_interchange_format(output_folder / output_filename,
                                            ds_all.pr.to_interchange_format())
 
-        # write data in native PRIAMP2 formart
+        # write data in native PRIAMP2 format
         encoding = {var: compression for var in ds_all.data_vars}
         ds_all.pr.to_netcdf(output_folder / (output_filename + ".nc"),
                               encoding=encoding)
@@ -274,9 +275,10 @@ def read_crf_for_country_datalad(
 
     print(f"Run the script using datalad run via the python api")
     script = code_path / "UNFCCC_CRF_reader" / "read_UNFCCC_CRF_submission.py"
+    #try:
     datalad.api.run(
-        cmd=f"./venv/bin/python3 {script.name} --country={country} "
-            f"--submission_year={submission_year} --submission_date=={submission_date}",
+        cmd=f"./venv/bin/python3 {script.as_posix()} --country={country} "
+            f"--submission_year={submission_year} --submission_date={submission_date}",
         dataset=root_path,
         message=f"Read data for {country}, CRF{submission_year}, {submission_date}.",
         inputs=input_files,
@@ -284,8 +286,16 @@ def read_crf_for_country_datalad(
         dry_run=None,
         explicit=True,
     )
+    #except IncompleteResultsError as exce:
+    #    print(f"Code did not run successfully:")
+    #    print(exce.failed)
 
 
+# function to read all available data (or list of countries?)
+# make sure it works when not all countries have submitted data
+# give option to only read new data (no output yet), but also option to
+# read all data, e.g. when specifications have changed
+
 
 def get_country_name(
         country_code: str,

code/UNFCCC_CRF_reader/read_UNFCCC_CRF_submission.py

@@ -3,7 +3,7 @@ This script is a wrapper around the read_crf_for_country
 function such that it can be called from datalad
 """
 
-from . import read_crf_for_country
+from UNFCCC_CRF_reader_prod import read_crf_for_country
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -17,8 +17,11 @@ country = args.country
 submission_year = args.submission_year
 submission_date = args.submission_date
 
+if submission_date == 'None':
+    submission_date = None
+
 read_crf_for_country(
     country,
-    submission_year=submission_year,
+    submission_year=int(submission_year),
     submission_date=submission_date)
 

code/UNFCCC_CRF_reader/read_UNFCCC_CRF_submission_datalad.py

@@ -4,7 +4,7 @@ from doit in the current setup where doit runs on system python and
 not in the venv.
 """
 
-from . import read_crf_for_country_datalad
+from UNFCCC_CRF_reader_prod import read_crf_for_country_datalad
 import argparse
 
 parser = argparse.ArgumentParser()
@@ -18,7 +18,10 @@ country = args.country
 submission_year = args.submission_year
 submission_date = args.submission_date
 
+if submission_date == "None":
+        submission_date = None
+
 read_crf_for_country_datalad(
         country,
-        submission_year=submission_year,
+        submission_year=int(submission_year),
         submission_date=submission_date)

code/UNFCCC_CRF_reader/utils.py → code/UNFCCC_CRF_reader/util.py

@@ -1,7 +1,8 @@
 from pathlib import Path
 
 # 4 for use from nbs, fix
-root_path = Path(__file__).parents[3].absolute()
+root_path = Path(__file__).parents[2].absolute()
+root_path = root_path.resolve()
 log_path = root_path / "log"
 code_path = root_path / "code"
 downloaded_data_path = root_path / "downloaded_data" / "UNFCCC"

dodo.py

@@ -164,7 +164,7 @@ def task_read_unfccc_crf_submission():
         'actions': [f"./venv/bin/python code/UNFCCC_CRF_reader/read_UNFCCC_CRF_submission_datalad.py "
                     f"--country={read_config_crf['country']} "
                     f"--submission_year={read_config_crf['submission_year']} "
-                    f"--submission_date={read_config['submission_date']}"],
+                    f"--submission_date={read_config_crf['submission_date']}"],
         'verbosity': 2,
         'setup': ['setup_venv'],
     }