[DATALAD RUNCMD] Fetch BUR submissions

=== Do not change lines below ===
{
 "chain": [],
 "cmd": "./venv/bin/python UNFCCC_GHG_data/UNFCCC_downloader/fetch_submissions_bur.py",
 "dsid": "4d062170-604c-4efd-afbf-5ce7f97e0e63",
 "exit": 0,
 "extra_inputs": [],
 "inputs": [],
 "outputs": [
  "downloaded_data/UNFCCC/submissions-bur.csv"
 ],
 "pwd": "."
}
^^^ Do not change lines above ^^^

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/CRF2023_AUS_specification.py

@@ -761,105 +761,101 @@ CRF2023_AUS = {
             "unit_info": unit_info["default"],
         },
         "sector_mapping": [
-            ['Drop down list:', ['\IGNORE'], 0],
-            ['1.B.2.a.vi.1. Abandoned wells', ['1.B.2.a.6.1'], 1],
-            ['1.B.2.a.vi.2. Other (please specify)', ['1.B.1.a.6.2'], 1],
-            ['NA', ['\IGNORE'], 2],
-            ['1. B. 2. a. Oil(6)', ['1.B.2.a'], 0],
-            ['1. Exploration', ['1.B.2.a.1'], 1],
-            ['2. Production(7)', ['1.B.2.a.2'], 1],
-            ['3. Transport', ['1.B.2.a.3'], 1],
-            ['4. Refining/storage', ['1.B.2.a.4'], 1],
-            ['5. Distribution of oil products', ['1.B.2.a.5'], 1],
-            ['6. Other', ['1.B.2.a.6'], 1],
+            ['1. B. 2. a. Oil (7)', ['1.B.2.a'], 0],
+            ['1.B.2.a.i. Exploration', ['1.B.2.a.1'], 1],
+            ['1.B.2.a.ii. Production and upgrading (8)', ['1.B.2.a.2'], 1],
+            ['1.B.2.a.iii. Transport', ['1.B.2.a.3'], 1],
+            ['1.B.2.a.iv. Refining/storage', ['1.B.2.a.4'], 1],
+            ['1.B.2.a.v. Distribution of oil products', ['1.B.2.a.5'], 1],
+            ['1.B.2.a.vi. Other', ['1.B.2.a.6'], 1],
+            ['Drop down list:', ['\IGNORE'], 2],
+            ['1.B.2.a.vi.1. Abandoned wells', ['1.B.2.a.6.1'], 2],
+            ['1.B.2.a.vi.2. Other (please specify)', ['1.B.1.a.6.2'], 2],
+            ['NA', ['\IGNORE'], 3],
             ['1. B. 2. b. Natural gas', ['1.B.2.b'], 0],
-            ['1. Exploration', ['1.B.2.b.1'], 1],
-            ['2. Production(7)', ['1.B.2.b.2'], 1],
-            ['3. Processing', ['1.B.2.b.3'], 1],
-            ['4. Transmission and storage', ['1.B.2.b.4'], 1],
-            ['5. Distribution', ['1.B.2.b.5'], 1],
-            ['6. Other', ['1.B.2.b.6'], 1],
+            ['1.B.2.b.i. Exploration', ['1.B.2.b.1'], 1],
+            ['1.B.2.b.ii. Production and gathering (8)', ['1.B.2.b.2'], 1],
+            ['1.B.2.b.iii. Processing', ['1.B.2.b.3'], 1],
+            ['1.B.2.b.iv. Transmission and storage', ['1.B.2.b.4'], 1],
+            ['1.B.2.b.v. Distribution', ['1.B.2.b.5'], 1],
+            ['1.B.2.b.vi. Other', ['1.B.2.b.6'], 1],
+            ['Drop down list:', ['\IGNORE'], 2],
+            ['1.B.2.b.vi.1. Gas post-meter', ['1.B.2.b.6.1'], 3],
+            ['1.B.2.b.vi.2. Abandoned wells', ['1.B.2.b.6.2'], 3],
+            ['1.B.2.b.vi.3. Other (please specify)', ['1.B.2.b.6.3'], 3],
+            ['LNG Terminals', ['1.B.2.b.6.3.a'], 4],
+            ['LNG Storage', ['1.B.2.b.6.3.b'], 4],
+            ['Natural Gas Storage', ['1.B.2.b.6.3.c'], 4],
             ['1. B. 2. c. Venting and flaring', ['1.B.2.c'], 0],
-            ['Venting', ['1.B.2.c-ven'], 1],
-            ['i. Oil', ['1.B.2.c-ven.i'], 2],
-            ['ii. Gas', ['1.B.2.c-ven.ii'], 2],
-            ['iii. Combined', ['1.B.2.c-ven.iii'], 2],
-            ['Flaring(8)', ['1.B.2.c-fla'], 1],
-            ['i. Oil', ['1.B.2.c-fla.i'], 2],
-            ['ii. Gas', ['1.B.2.c-fla.ii'], 2],
-            ['iii. Combined', ['1.B.2.c-fla.iii'], 2],
-            ['1.B.2.d. Other (please specify)(9)', ['1.B.2.d'], 0],
-            ['Groundwater extraction and CO2 mining', ['1.B.2.d.i'], 1],  # HUN
-            ['Geothermal', ['1.B.2.d.ii'], 1],  # NOR, DEU, PRT, NZL
-            ['Geothermal Energy', ['1.B.2.d.ii'], 1],  # ISL
-            ['Geothermal Generation', ['1.B.2.d.ii'], 1],  # JPN
-            ['Geotherm', ['1.B.2.d.ii'], 1],  # ITA
-            ['City Gas Production', ['1.B.2.d.iii'], 1],  # PRT
-            ['Other', ['1.B.2.d.iv'], 1],  # UKR, ROU
-            ['Other non-specified', ['1.B.2.d.iv'], 1],  # SWE
-            ['Flaring in refineries', ['1.B.2.d.v'], 1],  # ITA
-            ['LPG transport', ['1.B.2.d.vi'], 1],  # GRC
-            ['Distribution of town gas', ['1.B.2.d.vii'], 1],  # FIN
-            ['Petrol distribution', ['1.B.2.d.viii'], 1],  # IRL
-            ['Natural Gas Transport', ['1.B.2.d.ix'], 1],  # BLR
-            ['Natural gas exploration - N2O emissions', ['1.B.2.d.x'], 1],  # GBR, GBK
-            ['flue gas desulfurisation', ['1.B.2.d.xi'], 1],  # GBR, GBK, new in 2022
+            ['1.B.2.c.i. Venting', ['1.B.2.c-ven'], 1],
+            ['1.B.2.c.i.1. Oil', ['1.B.2.c-ven.i'], 2],
+            ['1.B.2.c.i.2. Gas', ['1.B.2.c-ven.ii'], 2],
+            ['1.B.2.c.i.3. Combined', ['1.B.2.c-ven.iii'], 2],
+            ['1.B.2.c.ii. Flaring (9)', ['1.B.2.c-fla'], 1],
+            ['1.B.2.c.ii.1. Oil', ['1.B.2.c-fla.i'], 2],
+            ['1.B.2.c.ii.2. Gas', ['1.B.2.c-fla.ii'], 2],
+            ['1.B.2.c.ii.3. Combined', ['1.B.2.c-fla.iii'], 2],
+            ['1.B.2.d. Other (please specify) (10)', ['1.B.2.d'], 0],
+            ['NA', ['\IGNORE'], 1],
         ],
         "entity_mapping": {
-            'EMISSIONS CH4 (4) Amount captured': 'CH4',
-            'EMISSIONS CO2 Emissions(3)': 'CO2',
-            'EMISSIONS N2O Amount captured': 'N2O',
+            'EMISSIONS CH4 (5)': 'CH4',
+            'EMISSIONS CO2 (4)': 'CO2',
+            'EMISSIONS N2O': 'N2O',
         },
         "coords_defaults": {
             "class": "Total",
         },
-    },  # tested
+    },  # to test
     "Table1.C": {
-        "status": "tested",
+        "status": "to_test",
         "table": {
-            "firstrow": 5,
-            "lastrow": 24,
+            "firstrow": 7,
+            "lastrow": 29,
             "header": ['group', 'entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category"],
             "cols_to_ignore": [
-                'ACTIVITY DATA CO2 transported or injected(1)',
+                'ACTIVITY DATA CO2 transported or injected (1)',
                 'IMPLIED EMISSION FACTORS CO2',
             ],
             "stop_cats": ["", np.nan],
             "unit_info": unit_info["default"],
         },
         "sector_mapping": [
-            ['1. Transport of CO2', ['1.C.1']],
-            ['a. Pipelines', ['1.C.1.a']],
-            ['b. Ships', ['1.C.1.b']],
-            ['c. Other', ['1.C.1.c']],
-            ['2. Injection and storage(3)', ['1.C.2']],
-            ['a. Injection', ['1.C.2.a']],
-            ['b. Storage', ['1.C.2.b']],
-            ['3. Other', ['1.C.3']],
-            ['Information item(4, 5)', ['\IGNORE']],
-            ['Total amount captured for storage', ['M.Info.A.TACS']],
-            ['Total amount of imports for storage', ['M.Info.A.TAIS']],
+            ['1.C.1. Transport of CO2', ['1.C.1'], 0],
+            ['1.C.1.a. Pipelines', ['1.C.1.a'], 1],
+            ['1.C.1.b. Ships', ['1.C.1.b'], 1],
+            ['1.C.1.c. Other', ['1.C.1.c'], 1],
+            ['NA', ['\IGNORE'], 2],
+            ['1.C.2. Injection and storage (3)', ['1.C.2'], 0],
+            ['1.C.2.a. Injection', ['1.C.2.a'], 1],
+            ['1.C.2.b. Storage', ['1.C.2.b'], 1],
+            ['1.C.3. Other', ['1.C.3'], 0],
+            ['NA', ['\IGNORE'], 1],
+            ['Information item (4, 5, 6)', ['\IGNORE']],
+            ['Total amount captured for storage (7)', ['M.Info.A.TACS']],
+            ['Total amount of imports for storage (7)', ['M.Info.A.TAIS']],
             ['Total A', ['M.Info.A']],
             ['Total amount of exports for storage', ['M.Info.B.TAES']],
             ['Total amount of CO2 injected at storage sites', ['M.Info.B.TAI']],
+            ['CO2 injected for operational usage (8)', ['M.Info.B.IOU']],
             ['Total leakage from transport, injection and storage', ['M.Info.B.TLTIS']],
             ['Total B', ['M.Info.B']],
             ['Difference (A-B)(6)', ['\IGNORE']],
         ],
         "entity_mapping": {
-            'EMISSIONS CO2(2)': 'CO2',
+            'EMISSIONS CO2 (2)': 'CO2',
         },
         "coords_defaults": {
             "class": "Total",
         },
-    },  # tested
+    },  # to_test
     "Table1.D": {
         "status": "TODO",
         "table": {
-            "firstrow": 5,
-            "lastrow": 20,
+            "firstrow": 7,
+            "lastrow": 24,
             "header": ['group', 'entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category", "class"],
@@ -875,59 +871,11 @@ CRF2023_AUS = {
             "class": "Total",
         },
     },  # TODO
-    "Table2(I)s1": {
-        "status": "tested",
-        "table": {
-            "firstrow": 5,
-            "lastrow": 31,
-            "header": ['entity', 'unit'],
-            "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
-            "categories": ["category"],
-            "cols_to_ignore": [],
-            "stop_cats": ["", np.nan],
-            "unit_info": unit_info["industry"],
-        },
-        "sector_mapping": [
-            ['Total industrial processes', ['2']],
-            ['A. Mineral industry', ['2.A']],
-            ['1. Cement production', ['2.A.1']],
-            ['2. Lime production', ['2.A.2']],
-            ['3. Glass production', ['2.A.3']],
-            ['4. Other process uses of carbonates', ['2.A.4']],
-            ['B. Chemical industry', ['2.B']],
-            ['1. Ammonia production', ['2.B.1']],
-            ['2. Nitric acid production', ['2.B.2']],
-            ['3. Adipic acid production', ['2.B.3']],
-            ['4. Caprolactam, glyoxal and glyoxylic acid production', ['2.B.4']],
-            ['5. Carbide production', ['2.B.5']],
-            ['6. Titanium dioxide production', ['2.B.6']],
-            ['7. Soda ash production', ['2.B.7']],
-            ['8. Petrochemical and carbon black production', ['2.B.8']],
-            ['9. Fluorochemical production', ['2.B.9']],
-            ['10. Other (as specified in table 2(I).A-H)', ['2.B.10']],
-            ['C. Metal industry', ['2.C']],
-            ['1. Iron and steel production', ['2.C.1']],
-            ['2. Ferroalloys production', ['2.C.2']],
-            ['3. Aluminium production', ['2.C.3']],
-            ['4. Magnesium production', ['2.C.4']],
-            ['5. Lead production', ['2.C.5']],
-            ['6. Zinc production', ['2.C.6']],
-            ['7. Other (as specified in table 2(I).A-H)', ['2.C.7']],
-        ],
-        "entity_mapping": {
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
-            'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
-        },
-        "coords_defaults": {
-            "class": "Total",
-        },
-    },  # tested
-    "Table2(I)s2": {
-        "status": "tested",
+    "Table2(I)": {
+        "status": "to_test",
         "table": {
-            "firstrow": 5,
-            "lastrow": 29,
+            "firstrow": 8,
+            "lastrow": 58,
             "header": ['entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category"],
@@ -936,84 +884,70 @@ CRF2023_AUS = {
             "unit_info": unit_info["industry"],
         },
         "sector_mapping": [
-            ['D. Non-energy products from fuels and solvent use', ['2.D']],
-            ['1. Lubricant use', ['2.D.1']],
-            ['2. Paraffin wax use', ['2.D.2']],
-            ['3. Other', ['2.D.3']],
-            ['E. Electronics industry', ['2.E']],
-            ['1. Integrated circuit or semiconductor', ['2.E.1']],
-            ['2. TFT flat panel display', ['2.E.2']],
-            ['3. Photovoltaics', ['2.E.3']],
-            ['4. Heat transfer fluid', ['2.E.4']],
-            ['5. Other (as specified in table 2(II))', ['2.E.5']],
-            ['F. Product uses as substitutes for ODS(2)', ['2.F']],
-            ['1. Refrigeration and air conditioning', ['2.F.1']],
-            ['2. Foam blowing agents', ['2.F.2']],
-            ['3. Fire protection', ['2.F.3']],
-            ['4. Aerosols', ['2.F.4']],
-            ['5. Solvents', ['2.F.5']],
-            ['6. Other applications', ['2.F.6']],
-            ['G. Other product manufacture and use', ['2.G']],
-            ['1. Electrical equipment', ['2.G.1']],
-            ['2. SF6 and PFCs from other product use', ['2.G.2']],
-            ['3. N2O from product uses', ['2.G.3']],
-            ['4. Other', ['2.G.4']],
-            ['H. Other (as specified in tables 2(I).A-H and 2(II))(3)', ['2.H']],
+            ['2. Total industrial processes', ['2']],
+            ['2.A. Mineral industry', ['2.A']],
+            ['2.A.1. Cement production', ['2.A.1']],
+            ['2.A.2. Lime production', ['2.A.2']],
+            ['2.A.3. Glass production', ['2.A.3']],
+            ['2.A.4. Other process uses of carbonates', ['2.A.4']],
+            ['2.B. Chemical industry', ['2.B']],
+            ['2.B.1. Ammonia production', ['2.B.1']],
+            ['2.B.2. Nitric acid production', ['2.B.2']],
+            ['2.B.3. Adipic acid production', ['2.B.3']],
+            ['2.B.4. Caprolactam, glyoxal and glyoxylic acid production', ['2.B.4']],
+            ['2.B.5. Carbide production', ['2.B.5']],
+            ['2.B.6. Titanium dioxide production', ['2.B.6']],
+            ['2.B.7. Soda ash production', ['2.B.7']],
+            ['2.B.8. Petrochemical and carbon black production', ['2.B.8']],
+            ['2.B.9. Fluorochemical production', ['2.B.9']],
+            ['2.B.10. Other', ['2.B.10']],
+            ['2.C. Metal industry', ['2.C']],
+            ['2.C.1. Iron and steel production', ['2.C.1']],
+            ['2.C.2. Ferroalloys production', ['2.C.2']],
+            ['2.C.3. Aluminium production', ['2.C.3']],
+            ['2.C.4. Magnesium production', ['2.C.4']],
+            ['2.C.5. Lead production', ['2.C.5']],
+            ['2.C.6. Zinc production', ['2.C.6']],
+            ['2.C.7. Other', ['2.C.7']],
+            ['2.D. Non-energy products from fuels and solvent use (4)', ['2.D']],
+            ['2.D.1. Lubricant use', ['2.D.1']],
+            ['2.D.2. Paraffin wax use', ['2.D.2']],
+            ['2.D.3. Other', ['2.D.3']],
+            ['2.E. Electronics industry', ['2.E']],
+            ['2.E.1. Integrated circuit or semiconductor', ['2.E.1']],
+            ['2.E.2. TFT flat panel display', ['2.E.2']],
+            ['2.E.3. Photovoltaics', ['2.E.3']],
+            ['2.E.4. Heat transfer fluid', ['2.E.4']],
+            ['2.E.5. Other (as specified in table 2(II))', ['2.E.5']],
+            ['2.F. Product uses as substitutes for ODS(2)', ['2.F']],
+            ['2.F.1. Refrigeration and air conditioning', ['2.F.1']],
+            ['2.F.2. Foam blowing agents', ['2.F.2']],
+            ['2.F.3. Fire protection', ['2.F.3']],
+            ['2.F.4. Aerosols', ['2.F.4']],
+            ['2.F.5. Solvents', ['2.F.5']],
+            ['2.F.6. Other applications', ['2.F.6']],
+            ['2.G. Other product manufacture and use', ['2.G']],
+            ['2.G.1. Electrical equipment', ['2.G.1']],
+            ['2.G.2. SF6 and PFCs from other product use', ['2.G.2']],
+            ['2.G.3. N2O from product uses', ['2.G.3']],
+            ['2.G.4. Other', ['2.G.4']],
+            ['2.H. Other (please specify) (5)', ['2.H']],
+            ['2.H.2. Food and Beverages Industry', ['2.H.2']],
         ],
         "entity_mapping": {
-            'HFCs(1)': 'HFCS (AR4GWP100)',
-            'PFCs(1)': 'PFCS (AR4GWP100)',
+            'HFCs (1)': 'HFCS (AR4GWP100)',
+            'PFCs (1)': 'PFCS (AR4GWP100)',
             'Unspecified mix of HFCs and PFCs(1)': 'UnspMixOfHFCsPFCs (AR4GWP100)',
         },
         "coords_defaults": {
             "class": "Total",
         },
-    },  # tested
-    "Table2(I).A-Hs1": {
-        "status": "TODO",
-        "table": {
-            "firstrow": 5,
-            "lastrow": 40,
-            "header": ['group', 'entity', 'entity', 'unit'],
-            "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
-            "categories": ["category"],
-            "cols_to_ignore": [],
-            "stop_cats": ["", np.nan],
-            "unit_info": unit_info["default"],
-        },
-        "sector_mapping": [
-
-        ],
-        "entity_mapping": [],
-        "coords_defaults": {
-            "class": "Total",
-        },
-    },  # TODO
-    "Table2(I).A-Hs2": {
-        "status": "TODO",
-        "table": {
-            "firstrow": 5,
-            "lastrow": 36,
-            "header": ['group', 'entity', 'entity', 'unit'],
-            "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
-            "categories": ["category"],
-            "cols_to_ignore": [],
-            "stop_cats": ["", np.nan],
-            "unit_info": unit_info["default"],
-        },
-        "sector_mapping": [
-
-        ],
-        "entity_mapping": [],
-        "coords_defaults": {
-            "class": "Total",
-        },
-    },  # TODO
+    },  # to test
     "Table2(II)": {
-        "status": "tested",
+        "status": "to_test",
         "table": {
-            "firstrow": 5,
-            "lastrow": 38,
+            "firstrow": 8,
+            "lastrow": 37, # ignore the totals
             "header": ['entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND SINK CATEGORIES",
             "categories": ["category"],
@@ -1022,40 +956,41 @@ CRF2023_AUS = {
             "unit_info": unit_info["fgases"],
         },
         "sector_mapping": [
-            ['Total actual emissions of halocarbons (by chemical) and SF6', ['2']],
-            ['B. Chemical industry', ['2.B']],
-            ['9. Flurochemical production', ['2.B.9']],
-            ['By-product emissions', ['2.B.9.a']],
-            ['Fugitive emissions', ['2.B.9.b']],
-            ['10. Other', ['2.B.10']],
-            ['C. Metal industry', ['2.C']],
-            ['3. Aluminium production', ['2.C.3']],
-            ['4. Magnesium production', ['2.C.4']],
-            ['7. Other', ['2.C.7']],
-            ['E. Electronics industry', ['2.E']],
-            ['1. Integrated circuit or semiconductor', ['2.E.1']],
-            ['2. TFT flat panel display', ['2.E.2']],
-            ['3. Photovoltaics', ['2.E.3']],
-            ['4. Heat transfer fluid', ['2.E.4']],
-            ['5. Other (as specified in table 2(II))', ['2.E.5']],
-            ['F. Product uses as substitutes for ODS(2)', ['2.F']],
-            ['1. Refrigeration and air conditioning', ['2.F.1']],
-            ['2. Foam blowing agents', ['2.F.2']],
-            ['3. Fire protection', ['2.F.3']],
-            ['4. Aerosols', ['2.F.4']],
-            ['5. Solvents', ['2.F.5']],
-            ['6. Other applications', ['2.F.6']],
-            ['G. Other product manufacture and use', ['2.G']],
-            ['1. Electrical equipment', ['2.G.1']],
-            ['2. SF6 and PFCs from other product use', ['2.G.2']],
-            ['4. Other', ['2.G.4']],
-            ['H. Other (please specify)', ['2.H']],
+            ['2. Total actual emissions of halocarbons (by chemical), SF6 and NF3',
+             ['2']],
+            ['2.B. Chemical industry', ['2.B']],
+            ['2.B.9. Flurochemical production', ['2.B.9']],
+            ['2.B.9.a. By-product emissions', ['2.B.9.a']],
+            ['2.B.9.b. Fugitive emissions', ['2.B.9.b']],
+            ['2.B.10. Other', ['2.B.10']],
+            ['2.C. Metal industry', ['2.C']],
+            ['2.C.3. Aluminium production', ['2.C.3']],
+            ['2.C.4. Magnesium production', ['2.C.4']],
+            ['2.C.7. Other', ['2.C.7']],
+            ['2.E. Electronics industry', ['2.E']],
+            ['2.E.1. Integrated circuit or semiconductor', ['2.E.1']],
+            ['2.E.2. TFT flat panel display', ['2.E.2']],
+            ['2.E.3. Photovoltaics', ['2.E.3']],
+            ['2.E.4. Heat transfer fluid', ['2.E.4']],
+            ['2.E.5. Other', ['2.E.5']],
+            ['2.F. Product uses as substitutes for ODS', ['2.F']],
+            ['2.F.1. Refrigeration and air conditioning', ['2.F.1']],
+            ['2.F.2. Foam blowing agents', ['2.F.2']],
+            ['2.F.3. Fire protection', ['2.F.3']],
+            ['2.F.4. Aerosols', ['2.F.4']],
+            ['2.F.5. Solvents', ['2.F.5']],
+            ['2.F.6. Other applications', ['2.F.6']],
+            ['2.G. Other product manufacture and use', ['2.G']],
+            ['2.G.1. Electrical equipment', ['2.G.1']],
+            ['2.G.2. SF6 and PFCs from other product use', ['2.G.2']],
+            ['2.G.4. Other', ['2.G.4']],
+            ['2.H. Other (please specify)', ['2.H']],
             ['2.H.1 Pulp and paper', ['2.H.1']],
             ['2.H.2 Food and beverages industry', ['2.H.2']],
             ['2.H.3 Other (please specify)', ['2.H.3']],
         ],
         "entity_mapping": {
-            'C 3F8': 'C3F8',
+            #'C3F8': 'C3F8',
             #'C10F18' 'C2F6' 'C4F10' 'C5F12' 'C6F14' 'CF4'
             'HFC-125': 'HFC125',
             'HFC-134': 'HFC134',
@@ -1085,12 +1020,12 @@ CRF2023_AUS = {
         "coords_defaults": {
             "class": "Total",
         },
-    },  # tested
-    "Table3s1": {  # Agriculture summary sheet 1
-        "status": "tested",
+    },  # to test
+    "Table3": {  # Agriculture summary
+        "status": "to_test",
         "table": {
-            "firstrow": 5,
-            "lastrow": 75,
+            "firstrow": 8,
+            "lastrow": 48,
             "header": ['entity', 'unit'],
             "col_for_categories": "GREENHOUSE GAS SOURCE AND",
             "categories": ["category"],
@@ -1100,205 +1035,53 @@ CRF2023_AUS = {
         },
         "sector_mapping": [
             ['3. Total agriculture', ['3'], 0],
-            # I. Livestock
-            ['I. Livestock', ['M.3.LV'], 1],
             # A. Enteric fermentation
-            ['A. Enteric fermentation', ['3.A'], 2],
-            ['1. Cattle(1)', ['3.A.1'], 3],
+            ['3.A. Enteric fermentation', ['3.A'], 2],
             ['Option A:', ['\IGNORE'], 4],
-            ['Dairy cattle', ['3.A.1.Aa'], 5],
-            ['Non-dairy cattle', ['3.A.1.Ab'], 5],
-            ['Option B:', ['\IGNORE'], 4],
-            ['Mature dairy cattle', ['3.A.1.Ba'], 5],
-            ['Other mature cattle', ['3.A.1.Bb'], 5],
-            ['Growing cattle', ['3.A.1.Bc'], 5],
-            ['Option C (country-specific):', ['\IGNORE'], 4],
-            # all countries not specified explcitly
-            ['\C!-AUS-MLT-LUX-POL-SVN-USA\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            # Australia
-            ['\C-AUS\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-AUS\ Dairy Cattle', ['3.A.1.C-AUS-a'], 6],
-            ['\C-AUS\ Beef Cattle - Pasture', ['3.A.1.C-AUS-b'], 6],
-            ['\C-AUS\ Beef Cattle - Feedlot', ['3.A.1.C-AUS-c'], 6],
-            # malta
-            ['\C-MLT\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-MLT\ dairy cows', ['3.A.1.C-MLT-a'], 6],
-            ['\C-MLT\ non-lactating cows', ['3.A.1.C-MLT-b'], 6],
-            ['\C-MLT\ bulls', ['3.A.1.C-MLT-c'], 6],
-            ['\C-MLT\ calves', ['3.A.1.C-MLT-d'], 6],
-            ['\C-MLT\ growing cattle 1-2 years', ['3.A.1.C-MLT-e'], 6],
-            # Luxembourg
-            ['\C-LUX\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-LUX\ Bulls', ['3.A.1.C-LUX-a'], 6],
-            ['\C-LUX\ Calves', ['3.A.1.C-LUX-b'], 6],
-            ['\C-LUX\ Young Cattle', ['3.A.1.C-LUX-c'], 6],
-            ['\C-LUX\ Suckler Cows', ['3.A.1.C-LUX-d'], 6],
-            ['\C-LUX\ Bulls under 2 years', ['3.A.1.C-LUX-e'], 6],
-            ['\C-LUX\ Dairy Cows', ['3.A.1.C-LUX-f'], 6],
-            # Poland
-            ['\C-POL\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-POL\ Bulls (older than 2 years)', ['3.A.1.C-POL-a'], 6],
-            ['\C-POL\ Non-dairy Heifers (older than 2 years)', ['3.A.1.C-POL-b'], 6],
-            ['\C-POL\ Non-dairy Young Cattle (younger than 1 year)', ['3.A.1.C-POL-c'], 6],
-            ['\C-POL\ Dairy Cattle', ['3.A.1.C-POL-d'], 6],
-            ['\C-POL\ Non-dairy Young Cattle (1-2 years)', ['3.A.1.C-POL-e'], 6],
-            # Slovenia
-            ['\C-SVN\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-SVN\ Dairy cows', ['3.A.1.C-SVN-a'], 6],
-            ['\C-SVN\ Non-dairy cattle', ['3.A.1.C-SVN-b'], 6],
-            ['\C-SVN\ Other cows', ['3.A.1.C-SVN-c'], 6],
-            # USA
-            ['\C-USA\ Other (as specified in table 3(I).A)', ['3.A.1.C'], 5],
-            ['\C-USA\ Steer Stocker', ['3.A.1.C-USA-a'], 6],
-            ['\C-USA\ Heifer Stocker', ['3.A.1.C-USA-b'], 6],
-            ['\C-USA\ Beef Cows', ['3.A.1.C-USA-c'], 6],
-            ['\C-USA\ Dairy Replacements', ['3.A.1.C-USA-d'], 6],
-            ['\C-USA\ Beef Replacements', ['3.A.1.C-USA-e'], 6],
-            ['\C-USA\ Steer Feedlot', ['3.A.1.C-USA-f'], 6],
-            ['\C-USA\ Heifer Feedlot', ['3.A.1.C-USA-g'], 6],
-            ['\C-USA\ Bulls', ['3.A.1.C-USA-h'], 6],
-            ['\C-USA\ Dairy Cows', ['3.A.1.C-USA-i'], 6],
-            ['\C-USA\ Beef Calves', ['3.A.1.C-USA-j'], 6],
-            ['\C-USA\ Dairy Calves', ['3.A.1.C-USA-k'], 6],
+            ['3.A.1.a Dairy cattle', ['3.A.1.Aa'], 5],
+            ['3.A.1.b Non-dairy cattle', ['3.A.1.Ab'], 5],
+            ['Option B (country-specific):', ['\IGNORE'], 4],
+            ['3.A.1.a Other', ['3.A.1.C'], 5],
             # Other livestock
-            ['2. Sheep', ['3.A.2'], 3],
-            ['3. Swine', ['3.A.3'], 3],
-            ['4. Other livestock', ['3.A.4'], 3],
-            ['Buffalo', ['3.A.4.a'], 4],
-            ['Camels', ['3.A.4.b'], 4],
-            ['Deer', ['3.A.4.c'], 4],
-            ['Goats', ['3.A.4.d'], 4],
-            ['Horses', ['3.A.4.e'], 4],
-            ['Mules and Asses', ['3.A.4.f'], 4],
-            ['Poultry', ['3.A.4.g'], 4],
-            ['Other (please specify)', ['3.A.4.h'], 4],
-            ['Rabbit', ['3.A.4.h.i'], 5],
-            ['Reindeer', ['3.A.4.h.ii'], 5],
-            ['Ostrich', ['3.A.4.h.iii'], 5],
-            ['Fur-bearing Animals', ['3.A.4.h.iv'], 5],
-            ['Other', ['3.A.4.h.v'], 5],
+            ['3.A.2. Sheep', ['3.A.2'], 3],
+            ['3.A.3. Swine', ['3.A.3'], 3],
+            ['3.A.4. Other livestock', ['3.A.4'], 3],
             # Manure Management
-            ['B. Manure management', ['3.B'], 2],
-            ['1. Cattle(1)', ['3.B.1'], 3],
+            ['3.B. Manure management', ['3.B'], 2],
+            ['3.B.1. Cattle(1)', ['3.B.1'], 3],
             ['Option A:', ['\IGNORE'], 4],
-            ['Dairy cattle', ['3.B.1.Aa'], 5],
-            ['Non-dairy cattle', ['3.B.1.Ab'], 5],
+            ['3.B.1.a. Dairy cattle', ['3.B.1.Aa'], 5],
+            ['3.B.1.b. Non-dairy cattle', ['3.B.1.Ab'], 5],
             ['Option B:', ['\IGNORE'], 4],
-            ['Mature dairy cattle', ['3.B.1.Ba'], 5],
-            ['Other mature cattle', ['3.B.1.Bb'], 5],
-            ['Growing cattle', ['3.B.1.Bc'], 5],
-            ['Option C (country-specific):', ['\IGNORE'], 4],
-            # all countries not specified explicitly
-            ['\C!-AUS-MLT-LUX-POL-SVN-USA\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            # Australia
-            ['\C-AUS\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-AUS\ Dairy Cattle', ['3.B.1.C-AUS-a'], 6],
-            ['\C-AUS\ Beef Cattle - Pasture', ['3.B.1.C-AUS-b'], 6],
-            ['\C-AUS\ Beef Cattle - Feedlot', ['3.B.1.C-AUS-c'], 6],
-            # Malta
-            ['\C-MLT\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-MLT\ dairy cows', ['3.B.1.C-MLT-a'], 6],
-            ['\C-MLT\ non-lactating cows', ['3.B.1.C-MLT-b'], 6],
-            ['\C-MLT\ bulls', ['3.B.1.C-MLT-c'], 6],
-            ['\C-MLT\ calves', ['3.B.1.C-MLT-d'], 6],
-            ['\C-MLT\ growing cattle 1-2 years', ['3.B.1.C-MLT-e'], 6],
-            # Luxembourg
-            ['\C-LUX\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-LUX\ Bulls', ['3.B.1.C-LUX-a'], 6],
-            ['\C-LUX\ Calves', ['3.B.1.C-LUX-b'], 6],
-            ['\C-LUX\ Young Cattle', ['3.B.1.C-LUX-c'], 6],
-            ['\C-LUX\ Suckler Cows', ['3.B.1.C-LUX-d'], 6],
-            ['\C-LUX\ Bulls under 2 years', ['3.B.1.C-LUX-e'], 6],
-            ['\C-LUX\ Dairy Cows', ['3.B.1.C-LUX-f'], 6],
-            # Poland
-            ['\C-POL\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-POL\ Non-dairy Cattle', ['3.B.1.C-POL-a'], 6],
-            ['\C-POL\ Dairy Cattle', ['3.B.1.C-POL-b'], 6],
-            # Slovenia
-            ['\C-SVN\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-SVN\ Dairy cows', ['3.B.1.C-SVN-a'], 6],
-            ['\C-SVN\ Non-dairy cattle', ['3.B.1.C-SVN-b'], 6],
-            ['\C-SVN\ Other cows', ['3.B.1.C-SVN-c'], 6],
-            # USA
-            ['\C-USA\ Other (as specified in table 3(I).B)', ['3.B.1.C'], 5],
-            ['\C-USA\ Dairy Cattle', ['\IGNORE'], 6],
-            ['\C-USA\ Non-Dairy Cattle', ['\IGNORE'], 6],
-            ['\C-USA\ Steer Stocker', ['3.B.1.C-USA-a'], 6],
-            ['\C-USA\ Heifer Stocker', ['3.B.1.C-USA-b'], 6],
-            ['\C-USA\ Beef Cows', ['3.B.1.C-USA-c'], 6],
-            ['\C-USA\ Dairy Replacements', ['3.B.1.C-USA-d'], 6],
-            ['\C-USA\ Beef Replacements', ['3.B.1.C-USA-e'], 6],
-            ['\C-USA\ Steer Feedlot', ['3.B.1.C-USA-f'], 6],
-            ['\C-USA\ Heifer Feedlot', ['3.B.1.C-USA-g'], 6],
-            ['\C-USA\ Bulls', ['3.B.1.C-USA-h'], 6],
-            ['\C-USA\ Dairy Cows', ['3.B.1.C-USA-i'], 6],
-            ['\C-USA\ Beef Calves', ['3.B.1.C-USA-j'], 6],
-            ['\C-USA\ Dairy Calves', ['3.B.1.C-USA-k'], 6],
-            # other animals
-            ['2. Sheep', ['3.B.2'], 3],
-            ['3. Swine', ['3.B.3'], 3],
-            ['4. Other livestock', ['3.B.4'], 3],
-            ['Buffalo', ['3.B.4.a'], 4],
-            ['Camels', ['3.B.4.b'], 4],
-            ['Deer', ['3.B.4.c'], 4],
-            ['Goats', ['3.B.4.d'], 4],
-            ['Horses', ['3.B.4.e'], 4],
-            ['Mules and Asses', ['3.B.4.f'], 4],
-            ['Poultry', ['3.B.4.g'], 4],
-            ['Other (please specify)', ['3.B.4.h'], 4],
-            ['Rabbit', ['3.B.4.h.i'], 5],
-            ['Reindeer', ['3.B.4.h.ii'], 5],
-            ['Ostrich', ['3.B.4.h.iii'], 5],
-            ['Fur-bearing Animals', ['3.B.4.h.iv'], 5],
-            ['Other', ['3.B.4.h.v'], 5],
-            ['5. Indirect N2O emissions', ['3.B.5'], 3],
-        ],
-        "coords_defaults": {
-            "class": "Total",
-        },
-    },  # tested
-    "Table3s2": {  # Agriculture summary sheet 2
-        "status": "tested",
-        "table": {
-            "firstrow": 5,
-            "lastrow": 18,
-            "header": ['entity', 'unit'],
-            "col_for_categories": "GREENHOUSE GAS SOURCE AND",
-            "categories": ["category"],
-            "cols_to_ignore": [],
-            "stop_cats": [".", np.nan],
-            "unit_info": unit_info["default"],
-        },
-        "sector_mapping": [
-            ['C. Rice cultivation', ['3.C']],
-            ['D. Agricultural soils(2) (3) (4)', ['3.D']],
-            ['E. Prescribed burning of savannahs', ['3.E']],
-            ['E. Prescribed burning of savannas', ['3.E']],
-            ['F. Field burning of agricultural residues', ['3.F']],
-            ['G. Liming', ['3.G']],
-            ['H. Urea application', ['3.H']],
-            ['I. Other carbon-containing fertilizers', ['3.I']],
-            ['J. Other (please specify)', ['3.J']],
-            ['NOx from Manure Management', ['3.J.1']],
-            ['3.B NOx Emissions', ['3.J.1']],
-            ['NOx from 3B', ['3.J.1']],
-            ['NOX emissions from manure management', ['3.J.1']],
-            ['NOx from manure management', ['3.J.1']],
-            ['Other', ['3.J.2']],
-            ['Other UK emissions', ['3.J.2']],
-            ['Other non-specified', ['3.J.2']],
-            ['OTs and CDs - Livestock', ['3.J.3']],
-            ['OTs and CDs - soils', ['3.J.4']],
-            ['OTs and CDs - other', ['3.J.5']],
-            ['Digestate renewable raw material (storage of N)', ['3.J.6']],
-            ['Digestate renewable raw material (atmospheric deposition)', ['3.J.7']],
-            ['Digestate renewable raw material (storage of dry matter)', ['3.J.8']],
-            ['NOx from Livestock', ['3.J.9']],
+            ['3.B.1.a Other', ['3.B.1.C'], 5],
+            ['2.B.2. Sheep', ['3.B.2'], 3],
+            ['2.B.3. Swine', ['3.B.3'], 3],
+            ['2.B.4. Other livestock', ['3.B.4'], 3],
+            ['3.B.5. Indirect N2O emissions', ['3.B.5'], 3],
+            ['3.C. Rice cultivation', ['3.C']],
+            ['3.D. Agricultural soils(4,5)', ['3.D']],
+            ['3.D.1. Direct N2O emissions from managed soils', '3.D.a']
+            ['3.D.1.a. Inorganic N fertilizers', ['3.D.a.1']],
+            ['3.D.1.b. Organic N fertilizers', ['3.D.a.2']],
+            ['3.D.1.c. Urine and dung deposited by grazing animals', ['3.D.a.3']],
+            ['3.D.1.d. Crop residues', ['3.D.a.4']],
+            ['3.D.1.e. Mineralization/immobilization associated with loss/gain of '
+             'soil organic matter', ['3.D.a.5']],
+            ['3.D.1.f. Cultivation of organic soils (i.e. histosols)', ['3.D.a.6']],
+            ['3.D.1.g. Other', ['3.D.a.7']],
+            ['3.D.2. Indirect N2O Emissions from managed soils', ['3.D.b']],
+            ['3.E. Prescribed burning of savannahs', ['3.E']],
+            ['3.F. Field burning of agricultural residues', ['3.F']],
+            ['3.G. Liming', ['3.G']],
+            ['3.H. Urea application', ['3.H']],
+            ['3.I. Other carbon-containing fertilizers', ['3.I']],
+            ['3.J. Other (please specify)', ['3.J']],
+            ['NA', ['\IGNORE']],
         ],
         "coords_defaults": {
             "class": "Total",
         },
-    },  # tested
+    },  # to test
     "Table3.C": {  # rice cultivation details
         "status": "tested",
         "table": {

UNFCCC_GHG_data/UNFCCC_CRF_reader/crf_specifications/util.py

@@ -27,6 +27,7 @@ unit_info = {
         "manual_repl_unit": {
             "CO2 equivalent (kt)": "kt CO2eq",
             "CO2 equivalents (kt)": "kt CO2eq", # for AUS Table1
+            "CO2 equivalent (kt) (3)": "kt CO2eq", # for AUS, Table2(I)
         },
         "default_unit": "kt",
     },

downloaded_data/UNFCCC/submissions-bur.csv

@@ -1 +1 @@
-../../.git/annex/objects/WQ/Qk/MD5E-s48477--b0cd720e42e0a9120eb5a926849e6f22.csv/MD5E-s48477--b0cd720e42e0a9120eb5a926849e6f22.csv
+../../.git/annex/objects/P7/W1/MD5E-s48743--deb29ebd7db0702a01bd62171b337443.csv/MD5E-s48743--deb29ebd7db0702a01bd62171b337443.csv