[DATALAD RUNCMD] Fetch NC submissions

=== Do not change lines below ===
{
 "chain": [],
 "cmd": "./venv/bin/python UNFCCC_GHG_data/UNFCCC_downloader/fetch_submissions_nc.py",
 "dsid": "4d062170-604c-4efd-afbf-5ce7f97e0e63",
 "exit": 0,
 "extra_inputs": [],
 "inputs": [],
 "outputs": [
  "downloaded_data/UNFCCC/submissions-nc.csv"
 ],
 "pwd": "."
}
^^^ Do not change lines above ^^^

UNFCCC_GHG_data/UNFCCC_DI_reader/UNFCCC_DI_reader_config.py

@@ -332,6 +332,17 @@ cat_conversion = {
 di_processing_templates = {
     # templates fro the DI processing. Most processing rules will apply to several
     # versions. So we store them here and refer to them in the processing info dict
+    # general templates
+    'general': {
+        'copyUnspHFCUnspPFC': {
+            'basket_copy': {
+                'GWPs_to_add': ["AR4GWP100", "AR5GWP100", "AR6GWP100"],
+                'entities': ["UnspMixOfHFCs", "UnspMixOfPFCs"],
+                'source_GWP': gwp_to_use,
+            },
+        }
+    },
+    # country templates
     #AFG: not needed (newer data in BUR1), 2005, 2013 only
     #AGO: 2000, 2005 only (external key needed for some gases / sectors)
     'ALB': {
@@ -426,6 +437,11 @@ di_processing_templates = {
                     },
                 },
             },
+            'basket_copy': {
+                'GWPs_to_add': ["AR4GWP100", "AR5GWP100", "AR6GWP100"],
+                'entities': ["UnspMixOfHFCs", "UnspMixOfPFCs"],
+                'source_GWP': gwp_to_use,
+            },
         }
     },
     #AND: no data
@@ -1692,6 +1708,10 @@ di_processing_info = {
         'default': di_processing_templates['ARE']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['ARE']['DI2023-05-24'],
     },
+    'ARG': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'AZE': {
         'default': di_processing_templates['AZE']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['AZE']['DI2023-05-24'],
@@ -1708,6 +1728,10 @@ di_processing_info = {
         'default': di_processing_templates['BIH']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['BIH']['DI2023-05-24'],
     },
+    'BOL': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'BRB': {
         'default': di_processing_templates['BRB']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['BRB']['DI2023-05-24'],
@@ -1756,10 +1780,18 @@ di_processing_info = {
         'default': di_processing_templates['IND']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['IND']['DI2023-05-24'],
     },
+    'JAM': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'KEN': {
         'default': di_processing_templates['KEN']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['KEN']['DI2023-05-24'],
     },
+    'KOR': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'LCA': {
         'default': di_processing_templates['LCA']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['LCA']['DI2023-05-24'],
@@ -1772,10 +1804,18 @@ di_processing_info = {
         'default': di_processing_templates['MAR']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['MAR']['DI2023-05-24'],
     },
+    'MDA': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'MDG': {
         'default': di_processing_templates['MDG']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['MDG']['DI2023-05-24'],
     },
+    'MDV': {
+        'default': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+        'DI2023-05-24': di_processing_templates['general']['copyUnspHFCUnspPFC'],
+    },
     'MLI': {
         'default': di_processing_templates['MLI']['DI2023-05-24'],
         'DI2023-05-24': di_processing_templates['MLI']['DI2023-05-24'],
@@ -1834,44 +1874,4 @@ basket_copy_unspHFCPFC = {
 },
 
 
-gas_baskets = {
-    'HFCS (SARGWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
-                     'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
-                     'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
-                     'HFC407c', 'HFC410a', 'HFC4310mee', #'OTHERHFCS (SARGWP100)',
-                         'UnspMixOfHFCs (SARGWP100)'],
-    'HFCS (AR4GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
-                     'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
-                     'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
-                     'HFC407c', 'HFC410a', 'HFC4310mee', 'UnspMixOfHFCs (AR4GWP100)'],
-    'HFCS (AR5GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
-                      'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
-                      'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
-                      'HFC407c', 'HFC410a', 'HFC4310mee',
-                         'UnspMixOfHFCs (AR5GWP100)'],
-    'HFCS (AR6GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
-                      'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
-                      'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
-                      'HFC407c', 'HFC410a', 'HFC4310mee',
-                         'UnspMixOfHFCs (AR6GWP100)'],
-    'PFCS (SARGWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
-                      'UnspMixOfPFCs (SARGWP100)'],
-    'PFCS (AR4GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
-                      'UnspMixOfPFCs (AR4GWP100)'],
-    'PFCS (AR5GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
-                      'UnspMixOfPFCs (AR5GWP100)'],
-    'PFCS (AR6GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
-                      'UnspMixOfPFCs (AR6GWP100)'],
-    'FGASES (SARGWP100)': ['HFCS (SARGWP100)', 'PFCS (SARGWP100)', 'SF6', 'NF3'],
-    'FGASES (AR4GWP100)': ['HFCS (AR4GWP100)', 'PFCS (AR4GWP100)', 'SF6', 'NF3'],
-    'FGASES (AR5GWP100)':['HFCS (AR5GWP100)', 'PFCS (AR5GWP100)', 'SF6', 'NF3'],
-    'FGASES (AR6GWP100)':['HFCS (AR6GWP100)', 'PFCS (AR6GWP100)', 'SF6', 'NF3'],
-    'KYOTOGHG (SARGWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (SARGWP100)',
-                          'PFCS (SARGWP100)'],
-    'KYOTOGHG (AR4GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR4GWP100)',
-                          'PFCS (AR4GWP100)'],
-    'KYOTOGHG (AR5GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR5GWP100)',
-                            'PFCS (AR5GWP100)'],
-    'KYOTOGHG (AR6GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR6GWP100)',
-                            'PFCS (AR6GWP100)'],
-}
+

UNFCCC_GHG_data/UNFCCC_DI_reader/UNFCCC_DI_reader_proc.py

@@ -8,11 +8,10 @@ from typing import Optional, Dict, List, Union
 
 from .UNFCCC_DI_reader_config import di_processing_info
 from .UNFCCC_DI_reader_config import cat_conversion
-from .UNFCCC_DI_reader_config import gas_baskets
 from .util import NoDIDataError, nAI_countries
 from .util import DI_date_format
 
-from UNFCCC_GHG_data.helper import process_data_for_country
+from UNFCCC_GHG_data.helper import process_data_for_country, gas_baskets
 from .UNFCCC_DI_reader_helper import find_latest_DI_data
 from .UNFCCC_DI_reader_helper import determine_filename
 
@@ -125,6 +124,13 @@ def process_UNFCCC_DI_for_country(
     else:
         processing_info_country_scen = None
 
+    # fill net emissions from actual emissions where necessary (e.g. 24540 for
+    # individual fgases)
+    data_country = data_country.pr.set("measure", "Net emissions/removals",
+                                       data_country.pr.loc[
+                                           {"measure": ["Actual emissions"]}],
+                                       existing='fillna')
+
     # 3: map categories
     if country_code in nAI_countries:
         # conversion from BURDI to IPCC2006_PRIMAP needed

UNFCCC_GHG_data/helper/__init__.py

@@ -4,7 +4,7 @@ from .definitions import legacy_data_path
 from .definitions import downloaded_data_path, downloaded_data_path_UNFCCC
 from .definitions import dataset_path, dataset_path_UNFCCC
 from .definitions import custom_country_mapping, custom_folders
-from .definitions import GWP_factors
+from .definitions import GWP_factors, gas_baskets
 from .functions import get_country_code, get_country_name, convert_categories
 from .functions import create_folder_mapping, process_data_for_country, get_code_file
 from .functions import fix_rows
@@ -23,6 +23,7 @@ __all__ = [
     "custom_country_mapping",
     "custom_folders",
     "GWP_factors",
+    "gas_baskets",
     "get_country_code",
     "get_country_name",
     "convert_categories",

UNFCCC_GHG_data/helper/definitions.py

@@ -85,4 +85,46 @@ GWP_factors = {
         'UnspMixOfHFCs': 1.27,
         'UnspMixOfPFCs': 1.18,
     },
+}
+
+gas_baskets = {
+    'HFCS (SARGWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
+                     'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
+                     'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
+                     'HFC407c', 'HFC410a', 'HFC4310mee', #'OTHERHFCS (SARGWP100)',
+                         'UnspMixOfHFCs (SARGWP100)'],
+    'HFCS (AR4GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
+                     'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
+                     'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
+                     'HFC407c', 'HFC410a', 'HFC4310mee', 'UnspMixOfHFCs (AR4GWP100)'],
+    'HFCS (AR5GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
+                      'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
+                      'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
+                      'HFC407c', 'HFC410a', 'HFC4310mee',
+                         'UnspMixOfHFCs (AR5GWP100)'],
+    'HFCS (AR6GWP100)': ['HFC23', 'HFC32', 'HFC41', 'HFC125', 'HFC134',
+                      'HFC134a', 'HFC143',  'HFC143a', 'HFC152a', 'HFC227ea',
+                      'HFC236fa', 'HFC245ca', 'HFC245fa', 'HFC365mfc',  'HFC404a',
+                      'HFC407c', 'HFC410a', 'HFC4310mee',
+                         'UnspMixOfHFCs (AR6GWP100)'],
+    'PFCS (SARGWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
+                      'UnspMixOfPFCs (SARGWP100)'],
+    'PFCS (AR4GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
+                      'UnspMixOfPFCs (AR4GWP100)'],
+    'PFCS (AR5GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
+                      'UnspMixOfPFCs (AR5GWP100)'],
+    'PFCS (AR6GWP100)': ['C3F8', 'C4F10', 'CF4', 'C2F6', 'C6F14', 'C5F12', 'cC4F8',
+                      'UnspMixOfPFCs (AR6GWP100)'],
+    'FGASES (SARGWP100)': ['HFCS (SARGWP100)', 'PFCS (SARGWP100)', 'SF6', 'NF3'],
+    'FGASES (AR4GWP100)': ['HFCS (AR4GWP100)', 'PFCS (AR4GWP100)', 'SF6', 'NF3'],
+    'FGASES (AR5GWP100)':['HFCS (AR5GWP100)', 'PFCS (AR5GWP100)', 'SF6', 'NF3'],
+    'FGASES (AR6GWP100)':['HFCS (AR6GWP100)', 'PFCS (AR6GWP100)', 'SF6', 'NF3'],
+    'KYOTOGHG (SARGWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (SARGWP100)',
+                          'PFCS (SARGWP100)'],
+    'KYOTOGHG (AR4GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR4GWP100)',
+                          'PFCS (AR4GWP100)'],
+    'KYOTOGHG (AR5GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR5GWP100)',
+                            'PFCS (AR5GWP100)'],
+    'KYOTOGHG (AR6GWP100)': ['CO2', 'CH4', 'N2O', 'SF6', 'NF3', 'HFCS (AR6GWP100)',
+                            'PFCS (AR6GWP100)'],
 }