Fix KYOTOGHG for nigeria BUR2

UNFCCC_GHG_data/UNFCCC_reader/Nigeria/config_NGA_BUR2.py

@@ -6,14 +6,16 @@ tables_trends = {
         'area': ['177,430,450,142'],
         'cols': ['208,260,311,355,406'],
         'coords_defaults': {
-            'entity': f'KYOTOGHG ({gwp_to_use})',
             'unit': 'GgCO2eq',
         },
         'coords_cols': {
             "category": "Year",
+            "entity": "entity",
+        },
+        'copy_cols': {
+            # to: from
+            'entity': 'Year',
         },
-        #'remove_cols': ['Per capita emissions (t)',
-        #                'GDP emissions index (Year 2000 = 100)'],
         'coords_value_mapping': {
             "unit": "PRIMAP1",
             'category': {
@@ -23,6 +25,13 @@ tables_trends = {
                 'AFOLU': '3',
                 'Waste': '4',
             },
+            'entity': {
+                'Total emissions': f'KYOTOGHG emissions ({gwp_to_use})',
+                'Energy': f'KYOTOGHG ({gwp_to_use})',
+                'IPPU': f'KYOTOGHG ({gwp_to_use})',
+                'AFOLU': f'KYOTOGHG emissions ({gwp_to_use})',
+                'Waste': f'KYOTOGHG ({gwp_to_use})',
+            },
         },
         'label_rows': [0, 1, 2],
     },
@@ -37,12 +46,12 @@ tables_trends = {
         'coords_cols': {
             "entity": "Year",
         },
-        'remove_cols': ['Total GHG emissions (CO₂-eq)',
-                        'Removals (CO₂) (CO₂-eq)',
-                        'CO₂ (Gg)'],
+        'remove_cols': [],
         'coords_value_mapping': {
             "unit": "PRIMAP1",
             'entity': {
+                'Total GHG emissions (CO₂-eq)': f'KYOTOGHG emissions ({gwp_to_use})',
+                'Removals (CO₂) (CO₂-eq)': 'CO2 removals',
                 'Net emissions (CO₂-eq)': f'KYOTOGHG ({gwp_to_use})',
                 'CO₂ (Gg)': 'CO2 emissions',
                 'CH₄ (CO₂-eq)': f'CH4 ({gwp_to_use})',
@@ -56,23 +65,36 @@ tables_trends = {
         'area': ['122,760,496,472'],
         'cols': ['159,212,265,311,355,406,456'],
         'coords_defaults': {
-            'entity': 'CO2',
+            #'entity': 'CO2',
             'unit': 'Gg',
         },
         'coords_cols': {
             "category": "Year",
+            'entity': 'entity',
         },
         'remove_cols': ['Total emissions'],
+        'copy_cols': {
+            # to: from
+            'entity': 'Year',
+        },
         'coords_value_mapping': {
             "unit": "PRIMAP1",
             'category': {
                 'Total net emissions': '0',
                 'Energy': '1',
                 'IPPU': '2',
-                'AFOLU - emissions': 'M.3.EMI',
-                'AFOLU - removals': 'M.3.REM',
+                'AFOLU - emissions': '3',
+                'AFOLU - removals': '3',
                 'Waste': '4',
             },
+            'entity': {
+                'Total net emissions': 'CO2',
+                'Energy': 'CO2',
+                'IPPU': 'CO2',
+                'AFOLU - emissions': 'CO2 emissions',
+                'AFOLU - removals': 'CO2 removals',
+                'Waste': 'CO2',
+            },
         },
         'label_rows':  [0, 1, 2],
     },
@@ -354,8 +376,6 @@ processing_info_step1 = {
               'name': 'Product uses as Substitutes for Ozone Depleting Substances'},
         '2': {'sources': ['2.A', '2.B', '2.C', '2.D', '2.E', '2.F', '2.G'],
               'name': 'IPPU'}, # for HFCs, PFCs, SO2, SF6, N2O (all 0)
-        '3': {'sources': ['M.3.EMI', 'M.3.REM'],
-              'name': 'AFOLU'}, # for CO2
     },
 }
 
@@ -430,4 +450,9 @@ processing_info_step2 =  {
                          f'UnspMixOfHFCs ({gwp_to_use})'],
         },
     },
+    'basket_copy': {
+        'GWPs_to_add': ["SARGWP100", "AR4GWP100", "AR6GWP100"],
+        'entities': ["HFCS", "PFCS", "UnspMixOfHFCs"],
+        'source_GWP': gwp_to_use,
+    },
 }

UNFCCC_GHG_data/UNFCCC_reader/Nigeria/read_NGA_BUR2_from_pdf.py

@@ -3,6 +3,7 @@
 
 import pandas as pd
 import primap2 as pm2
+import xarray as xr
 import numpy as np
 import camelot
 import locale
@@ -146,6 +147,11 @@ for table in tables_trends.keys():
     # make sure we have str not a number format for the dates
     df_this_table.columns = df_this_table.columns.map(str)
 
+    # make copy of columns if a column is used twice for metadata
+    if 'copy_cols' in current_table.keys():
+        for col in current_table["copy_cols"]:
+            df_this_table[col] = df_this_table[current_table["copy_cols"][col]]
+
     current_table["coords_defaults"].update(coords_defaults)
     # convert to interchange format
     data_current_if = pm2.pm2io.convert_wide_dataframe_if(
@@ -158,14 +164,14 @@ for table in tables_trends.keys():
         convert_str=True,
         time_format='%Y',
     )
-# todo: convert to native format before merge
+
     data_current_pm2 = pm2.pm2io.from_interchange_format(data_current_if)
     if data_trend_pm2 is None:
         data_trend_pm2 = data_current_pm2
     else:
         data_trend_pm2 = data_trend_pm2.pr.merge(data_current_pm2)
 
-data_pm2 = data_inv_pm2.pr.merge(data_trend_pm2, tolerance=0.05) # some rounding in
+data_pm2 = data_inv_pm2.pr.merge(data_trend_pm2, tolerance=0.02) # some rounding in
 # trends needs higher tolerance
 
 data_if = data_pm2.pr.to_interchange_format()
@@ -176,12 +182,13 @@ data_if = data_pm2.pr.to_interchange_format()
 if not output_folder.exists():
     output_folder.mkdir()
 pm2.pm2io.write_interchange_format(
-    output_folder / (output_filename + coords_terminologies["category"] + "_raw"),
+    output_folder / (output_filename + coords_terminologies["category"] + "_raw_test"),
     data_if)
 
 encoding = {var: compression for var in data_pm2.data_vars}
 data_pm2.pr.to_netcdf(
-    output_folder / (output_filename + coords_terminologies["category"] + "_raw.nc"),
+    output_folder / (output_filename + coords_terminologies["category"] +
+                     "_raw_test.nc"),
     encoding=encoding)
 
 
@@ -189,10 +196,35 @@ data_pm2.pr.to_netcdf(
 data_proc_pm2 = data_pm2
 terminology_proc = coords_terminologies["category"]
 
+# combine CO2 emissions and removals
+temp_CO2 = data_proc_pm2[["CO2 emissions", "CO2 removals"]].pr.sum\
+    (dim="entity", skipna=True, min_count=1)
+data_proc_pm2["CO2"] = data_proc_pm2["CO2"].fillna(temp_CO2)
+
+# create net KYOTOGHG for 0 and 3
+data_proc_pm2["KYOTOGHG removals (AR5GWP100)"] \
+    = xr.full_like(data_proc_pm2["CO2 removals"],
+                   np.nan).pr.quantify(units="Gg CO2 / year")
+
+data_proc_pm2["KYOTOGHG removals (AR5GWP100)"].attrs = {"entity": "KYOTOGHG",
+                                                        "gwp_context": "AR5GWP100"}
+data_proc_pm2["KYOTOGHG removals (AR5GWP100)"] \
+    = data_proc_pm2.pr.gas_basket_contents_sum(
+    basket="KYOTOGHG removals (AR5GWP100)", basket_contents=['CO2 removals'],
+    skipna=True, min_count=1)
+temp_KYOTOGHG = data_proc_pm2[["KYOTOGHG emissions (AR5GWP100)",
+                               "KYOTOGHG removals (AR5GWP100)"]].pr.sum\
+    (dim="entity", skipna=True, min_count=1)
+data_proc_pm2["KYOTOGHG (AR5GWP100)"] \
+    = data_proc_pm2["KYOTOGHG (AR5GWP100)"].fillna(temp_KYOTOGHG)
+
+
 # actual processing
 data_proc_pm2 = process_data_for_country(
     data_proc_pm2,
-    entities_to_ignore=[],
+    entities_to_ignore=['CO2 emissions', 'CO2 removals',
+                        'KYOTOGHG emissions (AR5GWP100)',
+                        'KYOTOGHG removals (AR5GWP100)'],
     gas_baskets={},
     processing_info_country=processing_info_step1,
 )

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP.csv

@@ -1 +0,0 @@
-../../../.git/annex/objects/8g/6p/MD5E-s326249--d68e5b004dec03486a6dc32ef9aea0a8.csv/MD5E-s326249--d68e5b004dec03486a6dc32ef9aea0a8.csv

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP.csv

@@ -0,0 +1 @@
+/annex/objects/MD5E-s326249--d68e5b004dec03486a6dc32ef9aea0a8.csv

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP.nc

@@ -1 +0,0 @@
-../../../.git/annex/objects/zM/WV/MD5E-s230478--ac02eac5533a41041b14832a7eb4d6fa.nc/MD5E-s230478--ac02eac5533a41041b14832a7eb4d6fa.nc

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP.nc

@@ -0,0 +1 @@
+/annex/objects/MD5E-s230478--ac02eac5533a41041b14832a7eb4d6fa.nc

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP_raw.csv

@@ -1 +0,0 @@
-../../../.git/annex/objects/W3/F7/MD5E-s64362--7454d52b074fd6662190fc1a3cee469d.csv/MD5E-s64362--7454d52b074fd6662190fc1a3cee469d.csv

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP_raw.csv

@@ -0,0 +1 @@
+/annex/objects/MD5E-s64362--7454d52b074fd6662190fc1a3cee469d.csv

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP_raw.nc

@@ -1 +0,0 @@
-../../../.git/annex/objects/07/M6/MD5E-s133588--3484314fbafd74e3013e8eeea6a9cc10.nc/MD5E-s133588--3484314fbafd74e3013e8eeea6a9cc10.nc

extracted_data/UNFCCC/Nigeria/NGA_BUR2_2021_IPCC2006_PRIMAP_raw.nc

@@ -0,0 +1 @@
+/annex/objects/MD5E-s133588--3484314fbafd74e3013e8eeea6a9cc10.nc