combined CRF data, fixes to DI data processing specifications

UNFCCC_GHG_data/UNFCCC_CRF_reader/UNFCCC_CRF_reader_core.py

@@ -636,8 +636,10 @@ def get_crf_files(
                 input_files = input_files + \
                               filter_filenames(input_folder.glob("*.xlsx"),
                                                **file_filter)
-        else:
-            raise ValueError(f"Folder {input_folder} does not exist")
+        #else:
+        #    raise ValueError(f"Folder {input_folder} does not exist")
+    if len(input_files) == 0:
+        raise ValueError(f"No input files found in {country_folders}")
 
     # make sure no files is in the list twice (happens when multiple input folder
     # contain the same submission which is possible when the country name is changed)
@@ -971,8 +973,9 @@ def get_latest_date_for_country(
         else:
             dates = []
             for folder in country_folders:
-                dates = dates + get_submission_dates(
-                    downloaded_data_path_UNFCCC / folder / f"CRF{submission_year}", file_filter)
+                folder_submission = downloaded_data_path_UNFCCC / folder / f"CRF{submission_year}"
+                if folder_submission.exists():
+                    dates = dates + get_submission_dates(folder_submission, file_filter)
             submission_date = find_latest_date(dates)
     else:
         raise ValueError(f"No data folder found for country {country_code}. "

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_AUS_specification.py

@@ -1414,6 +1414,7 @@ CRF2023_AUS = {
             ['6. Other (please specify) (7)', ['6']],
             ['NA', ['\IGNORE']],
             ['', ['\IGNORE']],
+            [np.nan, ['\IGNORE']],
             ['Memo items: (8)', ['\IGNORE']],
             ['1.D.1. International bunkers', ['M.Memo.Int']],
             ['1.D.1.a. Aviation', ['M.Memo.Int.Avi']],

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_specification.py

@@ -42,6 +42,10 @@ TODO:
 import numpy as np
 from .util import unit_info
 
+# TODO: GWPs now differ by country. This has to be implemented (maybe giving
+#  gwp_to_use as a parameter to the specification)
+gwp_to_use = "AR5GWP100"
+
 CRF2023 = {
     "Table1s1": {
         "status": "tested",
@@ -1643,9 +1647,9 @@ CRF2023 = {
             ['7. Other (as specified in table 2(I).A-H)', ['2.C.7']],
         ],
         "entity_mapping": {
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
+            'HFCs(1)': f'HFCS ({gwp_to_use})',
+            'PFCs(1)': f'PFCS ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
         },
         "coords_defaults": {
             "class": "Total",
@@ -1689,9 +1693,9 @@ CRF2023 = {
             ['H. Other (as specified in tables 2(I).A-H and 2(II))(3)', ['2.H']],
         ],
         "entity_mapping": {
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
+            'HFCs(1)': f'HFCS ({gwp_to_use})',
+            'PFCs(1)': f'PFCS ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
         },
         "coords_defaults": {
             "class": "Total",
@@ -1804,9 +1808,9 @@ CRF2023 = {
             'HFC-365mfc': 'HFC365mfc',
             'HFC-41': 'HFC41',
             'HFC-43-10mee': 'HFC4310mee',
-            'Unspecified mix of HFCs (1)': 'UnspMixOfHFCs (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
-            'Unspecified mix of PFCs (1)': 'UnspMixOfPFCs (AR4GWP100)',
+            'Unspecified mix of HFCs (1)': f'UnspMixOfHFCs ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
+            'Unspecified mix of PFCs (1)': f'UnspMixOfPFCs ({gwp_to_use})',
             'c-C3F6': 'cC3F6',
             'c-C4F8': 'cC4F8',
         },
@@ -2584,9 +2588,9 @@ CRF2023 = {
         "entity_mapping": {
             'NOX': 'NOx',
             'Net CO2 emissions/removals': 'CO2',
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
+            'HFCs(1)': f'HFCS ({gwp_to_use})',
+            'PFCs(1)': f'PFCS ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
         },
         "coords_defaults": {
             "class": "Total",
@@ -2637,9 +2641,9 @@ CRF2023 = {
         "entity_mapping": {
             'NOX': 'NOx',
             'Net CO2 emissions/removals': 'CO2',
-            'HFCs (1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
+            'HFCs (1)': f'HFCS ({gwp_to_use})',
+            'PFCs(1)': f'PFCS ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
         },
         "coords_defaults": {
             "class": "Total",
@@ -2673,9 +2677,9 @@ CRF2023 = {
         "entity_mapping": {
             'NOX': 'NOx',
             'Net CO2 emissions/removals': 'CO2',
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
+            'HFCs(1)': f'HFCS ({gwp_to_use})',
+            'PFCs(1)': f'PFCS ({gwp_to_use})',
+            'Unspecified mix of HFCs and PFCs(1)': f'UnspMixOfHFCsPFCs ({gwp_to_use})',
         },
         "coords_defaults": {
             "class": "Total",

UNFCCC_GHG_data/UNFCCC_DI_reader/UNFCCC_DI_reader_config.py

@@ -1237,6 +1237,13 @@ di_processing_templates = {
     'IND': {
         'DI2023-05-24': { # 1994,2000, 2010, 2016. Subsectors doffer a bit especilly
             # for 1994 and for LULUCF data
+            'remove_ts': {
+                'agri, waste': { # inconsistent with other sources
+                    'category': ['2.C', '2', '15163', '24540'],
+                    #'entities': ['CH4', 'N2O', f'KYOTOGHG ({gwp_to_use})'],
+                    'time': ['1994'],
+                },
+            },
             'downscale': {
                 'sectors': { # for 1994
                     '1.A': {

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-08.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/9q/q4/MD5E-s34973513--516f6654a86e9bf4a981d8c1eb6aa2aa.csv/MD5E-s34973513--516f6654a86e9bf4a981d8c1eb6aa2aa.csv

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-08.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/p4/fw/MD5E-s60843916--a792562b679509edd60c972e8b895052.nc/MD5E-s60843916--a792562b679509edd60c972e8b895052.nc

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-08.yaml

@@ -0,0 +1,30 @@
+attrs:
+  references: https://unfccc.int/ghg-inventories-annex-i-parties/2023
+  rights: ''
+  contact: mail@johannes-guetschow.de
+  title: 'Data submitted in 2023 to the UNFCCC in the common reporting format (CRF)
+    by Australia. Submission date: 13042023'
+  comment: Read fom xlsx file by Johannes Gütschow
+  institution: United Nations Framework Convention on Climate Change (www.unfccc.int)
+  cat: category (CRF2013_2023)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  sec_cats:
+  - c
+  - l
+  - a
+  - s
+  - s
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - category (CRF2013_2023)
+  - scenario (PRIMAP)
+  - class
+  - provenance
+  - source
+  - area (ISO3)
+  - entity
+  - unit
+data_file: CRF2023_raw_2023-09-08.csv

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-10.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/Fm/5g/MD5E-s35655447--102667deeaffa8d8aa0b474e443bb2b4.csv/MD5E-s35655447--102667deeaffa8d8aa0b474e443bb2b4.csv

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-10.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/FZ/JM/MD5E-s66594322--6570218bfb04c3574fba7ac05faa1f6d.nc/MD5E-s66594322--6570218bfb04c3574fba7ac05faa1f6d.nc

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-10.yaml

@@ -0,0 +1,30 @@
+attrs:
+  references: https://unfccc.int/ghg-inventories-annex-i-parties/2023
+  rights: ''
+  contact: mail@johannes-guetschow.de
+  title: 'Data submitted in 2023 to the UNFCCC in the common reporting format (CRF)
+    by Australia. Submission date: 13042023'
+  comment: Read fom xlsx file by Johannes Gütschow
+  institution: United Nations Framework Convention on Climate Change (www.unfccc.int)
+  cat: category (CRF2013_2023)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  sec_cats:
+  - c
+  - l
+  - a
+  - s
+  - s
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - category (CRF2013_2023)
+  - scenario (PRIMAP)
+  - class
+  - provenance
+  - source
+  - area (ISO3)
+  - entity
+  - unit
+data_file: CRF2023_raw_2023-09-10.csv