CRF fixes and small fix for Israel BUR2; new CRF data

UNFCCC_GHG_data/UNFCCC_CRF_reader/UNFCCC_CRF_reader_devel.py

@@ -130,6 +130,24 @@ def read_year_to_test_specs(
                     # now convert to native PRIMAP2 format
                     ds_table_pm2 = pm2.pm2io.from_interchange_format(ds_table_if)
 
+                    # if individual data for emissions and removals / recovery exist combine
+                    # them
+                    if "CO2 removals" in ds_table_pm2.data_vars:
+                        # we can just sum to CO2 as in a table either net CO2 exists,
+                        # or separate removals and emissions
+                        ds_table_pm2["CO2"] = ds_table_pm2[["CO2 emissions", "CO2 removals"]].pr.sum(
+                            dim="entity", skipna=True, min_count=1
+                        )
+                        ds_table_pm2["CO2"].attrs["entity"] = "CO2"
+                    
+                    if "CH4 removals" in ds_table_pm2.data_vars:
+                        # we can just sum to CO2 as in a table either net CO2 exists,
+                        # or separate removals and emissions
+                        ds_table_pm2["CH4"] = ds_table_pm2[["CH4 emissions", "CH4 removals"]].pr.sum(
+                            dim="entity", skipna=True, min_count=1
+                        )
+                        ds_table_pm2["CH4"].attrs["entity"] = "CH4"
+
                     # combine per table DS
                     if ds_all is None:
                         ds_all = ds_table_pm2

UNFCCC_GHG_data/UNFCCC_CRF_reader/UNFCCC_CRF_reader_prod.py

@@ -157,6 +157,26 @@ def read_crf_for_country(
             # now convert to native PRIMAP2 format
             ds_table_pm2 = pm2.pm2io.from_interchange_format(ds_table_if)
 
+            # if individual data for emissions and removals / recovery exist combine
+            # them
+            if 'CO2 removals' in ds_table_pm2.data_vars:
+                # we can just sum to CO2 as in a table either net CO2 exists,
+                # or separate removals and emissions
+                ds_table_pm2["CO2"] = ds_table_pm2[["CO2 emissions",
+                                                "CO2 removals"]].pr.sum(
+                    dim="entity", skipna=True, min_count=1
+                )
+                ds_table_pm2["CO2"].attrs["entity"] = "CO2"
+
+            if 'CH4 removals' in ds_table_pm2.data_vars:
+                # we can just sum to CO2 as in a table either net CO2 exists,
+                # or separate removals and emissions
+                ds_table_pm2["CH4"] = ds_table_pm2[["CH4 emissions",
+                                                "CH4 removals"]].pr.sum(
+                    dim="entity", skipna=True, min_count=1
+                )
+                ds_table_pm2["CH4"].attrs["entity"] = "CH4"
+
             # combine per table DS
             if ds_all is None:
                 ds_all = ds_table_pm2

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_AUS_specification.py

@@ -739,10 +739,10 @@ CRF2023_AUS = {
             ['NA', ['\IGNORE'], 1],
         ],
         "entity_mapping": {
-            'EMISSIONS (1) CH4': 'CH4EMI',
-            'EMISSIONS (1) CO2': 'CO2EMI',
-            'RECOVERY/FLARING (2) CH4': 'CH4REC',
-            'RECOVERY/FLARING (2) CO2': 'CO2REC',
+            'EMISSIONS (1) CH4': 'CH4 emissions',
+            'EMISSIONS (1) CO2': 'CO2 emissions',
+            'RECOVERY/FLARING (2) CH4': 'CH4 removals',
+            'RECOVERY/FLARING (2) CO2': 'CO2 removals',
         },
         "coords_defaults": {
             "class": "Total",
@@ -807,9 +807,9 @@ CRF2023_AUS = {
         ],
         "entity_mapping": {
             'EMISSIONS CH4 (5)': 'CH4',
-            'EMISSIONS CO2 (4)': 'CO2EMI',
+            'EMISSIONS CO2 (4)': 'CO2 emissions',
             'EMISSIONS N2O': 'N2O',
-            "RECOVERY (2) CO2": 'CO2REC',
+            "RECOVERY (2) CO2": 'CO2 removals',
         },
         "coords_defaults": {
             "class": "Total",

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_specification.py

@@ -44,7 +44,7 @@ from .util import unit_info
 
 # TODO: GWPs now differ by country. This has to be implemented (maybe giving
 #  gwp_to_use as a parameter to the specification)
-gwp_to_use = "AR5GWP100"
+gwp_to_use = "AR4GWP100"
 
 CRF2023 = {
     "Table1s1": {

UNFCCC_GHG_data/UNFCCC_reader/Israel/config_ISR_BUR2.py

@@ -420,6 +420,11 @@ gas_baskets = {
     'KYOTOGHG (AR6GWP100)': ['CO2', 'CH4', 'N2O', 'FGASES (AR6GWP100)'],
 }
 
+basket_copy = {
+    'GWPs_to_add': ["AR4GWP100", "AR5GWP100", "AR6GWP100"],
+    'entities': ["HFCS", "PFCS"],
+    'source_GWP': gwp_to_use,
+}
 
 #### functions
 def is_int(input: str) -> bool:

UNFCCC_GHG_data/UNFCCC_reader/Israel/read_ISR_BUR2_from_pdf.py

@@ -16,7 +16,7 @@ from config_ISR_BUR2 import coords_cols, coords_terminologies, coords_defaults,
     coords_value_mapping, filter_remove, filter_keep, meta_data
 from config_ISR_BUR2 import cat_conversion, sectors_to_save, downscaling, \
     cats_to_agg, gas_baskets, terminology_proc
-from config_ISR_BUR2 import is_int
+from config_ISR_BUR2 import is_int, basket_copy
 
 ### genral configuration
 input_folder = downloaded_data_path / 'UNFCCC' / 'Israel' / 'BUR2'
@@ -267,7 +267,9 @@ data_proc_pm2 = process_data_for_country(
 
 country_processing_step2 = {
     'downscale': downscaling,
+    'basket_copy': basket_copy,
 }
+
 data_proc_pm2 = process_data_for_country(
     data_proc_pm2,
     entities_to_ignore=[],

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-13.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/77/gq/MD5E-s35690503--011e873a58d006d334f44ccc8ad69c28.csv/MD5E-s35690503--011e873a58d006d334f44ccc8ad69c28.csv

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-13.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/38/kw/MD5E-s66349302--45702253293b4622a5a71213aa3ebc7f.nc/MD5E-s66349302--45702253293b4622a5a71213aa3ebc7f.nc

datasets/UNFCCC/CRF2023/CRF2023_raw_2023-09-13.yaml

@@ -0,0 +1,30 @@
+attrs:
+  references: https://unfccc.int/ghg-inventories-annex-i-parties/2023
+  rights: ''
+  contact: mail@johannes-guetschow.de
+  title: 'Data submitted in 2023 to the UNFCCC in the common reporting format (CRF)
+    by Australia. Submission date: 13042023'
+  comment: Read fom xlsx file by Johannes Gütschow
+  institution: United Nations Framework Convention on Climate Change (www.unfccc.int)
+  cat: category (CRF2013_2023)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  sec_cats:
+  - c
+  - l
+  - a
+  - s
+  - s
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - provenance
+  - scenario (PRIMAP)
+  - category (CRF2013_2023)
+  - area (ISO3)
+  - class
+  - source
+  - entity
+  - unit
+data_file: CRF2023_raw_2023-09-13.csv