More small fixes to CRF reading code, specifications and folder mapping

code/UNFCCC_CRF_reader/UNFCCC_CRF_reader_core.py

@@ -252,16 +252,21 @@ def read_crf_table(
     unknown_rows = []
     last_row_info = []
     for file in input_files:
-        df_this_file, unknown_rows_this_file, last_row_info_this_file = \
-            read_crf_table_from_file(file, table, crf_spec[table])
-        if df_all is None:
-            df_all = df_this_file.copy(deep=True)
-            unknown_rows = unknown_rows_this_file
-            last_row_info = last_row_info_this_file
-        else:
-            df_all = pd.concat([df_this_file, df_all])
-            unknown_rows = unknown_rows + unknown_rows_this_file
-            last_row_info = last_row_info + last_row_info_this_file
+        file_info = get_info_from_crf_filename(file)
+        try:
+            int(file_info["data_year"])
+            df_this_file, unknown_rows_this_file, last_row_info_this_file = \
+                read_crf_table_from_file(file, table, crf_spec[table])
+            if df_all is None:
+                df_all = df_this_file.copy(deep=True)
+                unknown_rows = unknown_rows_this_file
+                last_row_info = last_row_info_this_file
+            else:
+                df_all = pd.concat([df_this_file, df_all])
+                unknown_rows = unknown_rows + unknown_rows_this_file
+                last_row_info = last_row_info + last_row_info_this_file
+        except:
+            print(f"Year could not be converted to int for file {file}. Skipping file.")
 
     return df_all, unknown_rows, last_row_info
 

code/UNFCCC_CRF_reader/UNFCCC_CRF_reader_prod.py

@@ -6,11 +6,11 @@
 #import pandas as pd
 import xarray as xr
 import primap2 as pm2
-import numpy as np
-import pycountry
+#import numpy as np
+#import pycountry
 import datalad.api
 from datetime import date
-from pathlib import Path
+#from pathlib import Path
 from typing import Optional, List, Dict, Union
 
 #from . import crf_specifications as crf

code/UNFCCC_CRF_reader/crf_specifications/CRF2021_specification.py

@@ -852,9 +852,9 @@ CRF2021 = {
             ['Gaseous Fuels', ['1.A.3.b.v.8', 'Gaseous'], 4],
             ['Other Liquid Fuels', ['1.A.3.b.v.9', 'Total'], 3],
             ['Other Liquid Fuels (please specify)', ['1.A.3.b.v.9', 'OtherLiquid'], 4],
-            ['Other Kerosene', ['1.A.3.b.v.9', 'OtherLiquid'], 5],
-            ['Heating and Other Gasoil', ['1.A.3.b.v.9', 'OtherLiquid'], 5],
-            ['Biomass', ['1.A.3.b.v.9', 'Total'], 3],
+            ['Other Kerosene', ['1.A.3.b.v.9', 'Kerosene'], 5],
+            ['Heating and Other Gasoil', ['1.A.3.b.v.9', 'HeatingGasoil'], 5],
+            ['Biomass', ['1.A.3.b.v.10', 'Total'], 3],
             ['Biomass', ['1.A.3.b.v.10', 'Biomass'], 4],
             # DEU
             ['CO2 from lubricant co-incineration in 2-stroke road vehicles', ['1.A.3.b.v.7', 'Total'], 3],
@@ -2251,7 +2251,7 @@ CRF2021 = {
             ['2. Industrial wastewater', ['5.D.2']],
             ['3. Other (as specified in table 5.D)', ['5.D.3']],
             ['E. Other (please specify)', ['5.E']],
-            ['Other', ['5.E']],  # EST, NOR
+            ['Other', ['5.E.5']],  # EST, NOR
             ['Recycling activities', ['5.E.1']],  # NLD
             ['Mechanical-Biological Treatment MBT', ['5.E.2']],  # DEU
             ['Accidental fires', ['5.E.3']],  # DEU, DKE, DNK, DNM

code/UNFCCC_CRF_reader/crf_specifications/CRF2022_specification.py

@@ -853,9 +853,9 @@ CRF2022 = {
             ['Gaseous Fuels', ['1.A.3.b.v.8', 'Gaseous'], 4],
             ['Other Liquid Fuels', ['1.A.3.b.v.9', 'Total'], 3],
             ['Other Liquid Fuels (please specify)', ['1.A.3.b.v.9', 'OtherLiquid'], 4],
-            ['Other Kerosene', ['1.A.3.b.v.9', 'OtherLiquid'], 5],
-            ['Heating and Other Gasoil', ['1.A.3.b.v.9', 'OtherLiquid'], 5],
-            ['Biomass', ['1.A.3.b.v.9', 'Total'], 3],
+            ['Other Kerosene', ['1.A.3.b.v.9', 'Kerosene'], 5],
+            ['Heating and Other Gasoil', ['1.A.3.b.v.9', 'HeatingGasoil'], 5],
+            ['Biomass', ['1.A.3.b.v.10', 'Total'], 3],
             ['Biomass', ['1.A.3.b.v.10', 'Biomass'], 4],
             # DEU
             ['CO2 from lubricant co-incineration in 2-stroke road vehicles', ['1.A.3.b.v.7', 'Total'], 3],
@@ -2268,7 +2268,7 @@ CRF2022 = {
             ['2. Industrial wastewater', ['5.D.2']],
             ['3. Other (as specified in table 5.D)', ['5.D.3']],
             ['E. Other (please specify)', ['5.E']],
-            ['Other', ['5.E']],  # EST, NOR
+            ['Other', ['5.E.5']],  # EST, NOR
             ['Recycling activities', ['5.E.1']],  # NLD
             ['Mechanical-Biological Treatment MBT', ['5.E.2']],  # DEU
             ['Accidental fires', ['5.E.3']],  # DEU, DKE, DNK, DNM

code/UNFCCC_reader/get_submissions_info.py

@@ -7,6 +7,9 @@ import json
 import pycountry
 #import os
 
+import sys
+sys.path.append(code_path.name)
+from UNFCCC_CRF_reader.util import custom_country_mapping
 
 def get_country_submissions(
         country_name: str,
@@ -521,13 +524,15 @@ def create_folder_mapping(
     rootpath = rootpath.resolve()
     folder = rootpath / folder
 
-    if extracted:
-        folder_mapping = {}
-    else:
+    folder_mapping = custom_country_mapping
+    if not extracted:
         folder_mapping = {
-            'VEN': 'Venezeula_(Bolivarian_Republic_of)',
-            'FSM': 'Micronesia_(Federated_State_of)',
-            'MKD': 'The_Republic_of_North_Macedonia',
+            **folder_mapping,
+            **{
+                'VEN': 'Venezeula_(Bolivarian_Republic_of)',
+                'FSM': 'Micronesia_(Federated_State_of)',
+                'MKD': 'The_Republic_of_North_Macedonia',
+            }
         }
     known_folders = list(folder_mapping.values())
 

dodo.py

@@ -207,7 +207,7 @@ def task_read_new_unfccc_crf_for_year():
     #if read_config_crf["countries"] is not None:
     #        actions[0] = actions[0] + f"--countries={read_config_crf['countries']} "
     if read_config_crf["re_read"]:
-        actions[0] = actions[0] + "--re_read"
+        actions[0] = actions[0] + " --re_read"
     return {
         #'basename': "Read_CRF_year",
         'actions': actions,