docs/source/api/unfccc_ghg_data.unfccc_reader.Bangladesh.config_bgd_bur1.rst

@@ -0,0 +1,6 @@
+unfccc\_ghg\_data.unfccc\_reader.Bangladesh.config\_bgd\_bur1
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: unfccc_ghg_data.unfccc_reader.Bangladesh.config_bgd_bur1
+
+.. currentmodule:: unfccc_ghg_data.unfccc_reader.Bangladesh.config_bgd_bur1

docs/source/api/unfccc_ghg_data.unfccc_reader.Bangladesh.read_BGD_BUR1_from_pdf.rst

@@ -0,0 +1,6 @@
+unfccc\_ghg\_data.unfccc\_reader.Bangladesh.read\_BGD\_BUR1\_from\_pdf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: unfccc_ghg_data.unfccc_reader.Bangladesh.read_BGD_BUR1_from_pdf
+
+.. currentmodule:: unfccc_ghg_data.unfccc_reader.Bangladesh.read_BGD_BUR1_from_pdf

docs/source/api/unfccc_ghg_data.unfccc_reader.Bangladesh.rst

@@ -0,0 +1,13 @@
+unfccc\_ghg\_data.unfccc\_reader.Bangladesh
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: unfccc_ghg_data.unfccc_reader.Bangladesh
+
+.. currentmodule:: unfccc_ghg_data.unfccc_reader.Bangladesh
+
+
+.. autosummary::
+  :toctree: ./
+
+  unfccc_ghg_data.unfccc_reader.Bangladesh.config_bgd_bur1
+  unfccc_ghg_data.unfccc_reader.Bangladesh.read_BGD_BUR1_from_pdf

docs/source/api/unfccc_ghg_data.unfccc_reader.rst

@@ -10,6 +10,7 @@ unfccc\_ghg\_data.unfccc\_reader
   :toctree: ./
 
   unfccc_ghg_data.unfccc_reader.Argentina
+  unfccc_ghg_data.unfccc_reader.Bangladesh
   unfccc_ghg_data.unfccc_reader.Burundi
   unfccc_ghg_data.unfccc_reader.Chile
   unfccc_ghg_data.unfccc_reader.China

downloaded_data/UNFCCC/00_new_downloads_BTR1-2024-09-01.csv

@@ -0,0 +1 @@
+../../.git/annex/objects/8x/F9/MD5E-s1564--0093464fa91a55abe8ff986b3f35262f.csv/MD5E-s1564--0093464fa91a55abe8ff986b3f35262f.csv

downloaded_data/UNFCCC/00_new_downloads_NC-2024-09-01.csv

@@ -0,0 +1 @@
+../../.git/annex/objects/M6/x9/MD5E-s500--6b20197d0296b0dda3efd62337283760.csv/MD5E-s500--6b20197d0296b0dda3efd62337283760.csv

downloaded_data/UNFCCC/Andorra/NC3/CN3_ANDORRA.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/Ww/Q4/MD5E-s5075032--2e69ab72f14331c8d508363d439699c1.pdf/MD5E-s5075032--2e69ab72f14331c8d508363d439699c1.pdf

downloaded_data/UNFCCC/Canada/BTR1/(Additional Information) CAN_2024_1990_to_2022_25032024.zip

@@ -0,0 +1 @@
+../../../../.git/annex/objects/7J/JQ/MD5E-s30195370--7632051405e2e6e9ba9d3929c258c7f3.zip/MD5E-s30195370--7632051405e2e6e9ba9d3929c258c7f3.zip

downloaded_data/UNFCCC/Canada/BTR1/2024NIR - Part 1.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/K7/2V/MD5E-s10492199--21908b648ee711472d67d20c4dd15b99.pdf/MD5E-s10492199--21908b648ee711472d67d20c4dd15b99.pdf

downloaded_data/UNFCCC/Canada/BTR1/2024NIR_-_Part_2.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/Q7/Q6/MD5E-s20079842--804a763cd62f6c8ebb986cfa52b52227.pdf/MD5E-s20079842--804a763cd62f6c8ebb986cfa52b52227.pdf

downloaded_data/UNFCCC/Canada/BTR1/2024NIR_-_Part_3.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/q5/kj/MD5E-s6004956--cd7ef3d42b234964bce5694464fdd700.pdf/MD5E-s6004956--cd7ef3d42b234964bce5694464fdd700.pdf

downloaded_data/UNFCCC/Canada/BTR1/2024RIN - Partie 1.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/p2/M1/MD5E-s10982889--3cafbb0243561ea49054277a206cd839.pdf/MD5E-s10982889--3cafbb0243561ea49054277a206cd839.pdf

downloaded_data/UNFCCC/Canada/BTR1/2024RIN_-_Partie_2.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/kW/jM/MD5E-s18139134--d8705e29773801b0376ec885588fc6ee.pdf/MD5E-s18139134--d8705e29773801b0376ec885588fc6ee.pdf

downloaded_data/UNFCCC/Canada/BTR1/2024RIN_-_Partie_3.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/pZ/gV/MD5E-s6675125--c6a39f397d558a1c3c4425347d69fcc9.pdf/MD5E-s6675125--c6a39f397d558a1c3c4425347d69fcc9.pdf

downloaded_data/UNFCCC/Canada/BTR1/can-2024-may-nid.zip

@@ -0,0 +1 @@
+../../../../.git/annex/objects/FX/0K/MD5E-s36232852--a9b028f334b85ee1b949deead7d26beb.zip/MD5E-s36232852--a9b028f334b85ee1b949deead7d26beb.zip

downloaded_data/UNFCCC/Canada/BTR1/can-2024-may-nid_%28fr%29.zip

@@ -0,0 +1 @@
+../../../../.git/annex/objects/J0/27/MD5E-s36550957--2cf7432a83f2808c0d657bc6d0255ba5.zip/MD5E-s36550957--2cf7432a83f2808c0d657bc6d0255ba5.zip

downloaded_data/UNFCCC/Comoros/NC3/troisi%C3%A8me_communication_nationale_des_Comores.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/zJ/p9/MD5E-s5297862--8a60561d9b3e3dca7aec4546540e9f0b.pdf/MD5E-s5297862--8a60561d9b3e3dca7aec4546540e9f0b.pdf

downloaded_data/UNFCCC/Guyana/NC3/Guyana_-_Third_National_Communication_to_the_UNFCCC_-_August_2024.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/0j/xX/MD5E-s20035525--6bce8ed4ee43803adb4da949c2d6ad82.pdf/MD5E-s20035525--6bce8ed4ee43803adb4da949c2d6ad82.pdf

downloaded_data/UNFCCC/Panama/BTR1/000_-_INFORME_BIENAL_-_DIGITAL_FINAL_compressed.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/Vp/5p/MD5E-s16289446--3d0e553b0be1c177a2636a81f630dc5b.pdf/MD5E-s16289446--3d0e553b0be1c177a2636a81f630dc5b.pdf

downloaded_data/UNFCCC/Panama/BTR1/2024_DIN_PA_vf.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/x0/qg/MD5E-s16067770--43421432a2ef917d154bf901167e8f59.pdf/MD5E-s16067770--43421432a2ef917d154bf901167e8f59.pdf

downloaded_data/UNFCCC/Uzbekistan/BTR1/NIR_Uzb_eng_26_06_2024_%D1%84%D0%B8%D0%BD%D0%B0%D0%BB.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/53/7X/MD5E-s4393471--2d0ab2f69dbd02775d49809cb6816a98.pdf/MD5E-s4393471--2d0ab2f69dbd02775d49809cb6816a98.pdf

downloaded_data/UNFCCC/Uzbekistan/BTR1/NIR_Uzb_rus_26__06_2024_%D1%84%D0%B8%D0%BD%D0%B0%D0%BB.pdf

@@ -0,0 +1 @@
+../../../../.git/annex/objects/30/V3/MD5E-s4367232--71a8903abcb1dd23d21073181aa5b715.pdf/MD5E-s4367232--71a8903abcb1dd23d21073181aa5b715.pdf

downloaded_data/UNFCCC/submissions-BTR1.csv

@@ -1 +1 @@
-../../.git/annex/objects/Z9/Qz/MD5E-s1734--60373097e7d45acd9783c02ebee355b4.csv/MD5E-s1734--60373097e7d45acd9783c02ebee355b4.csv
+../../.git/annex/objects/mF/81/MD5E-s4170--ebf216df90a32fc9af06099f928504fd.csv/MD5E-s4170--ebf216df90a32fc9af06099f928504fd.csv

downloaded_data/UNFCCC/submissions-nc.csv

@@ -1 +1 @@
-../../.git/annex/objects/p5/qp/MD5E-s84719--c96eb5f9833f3ae071eb551a15440b9e.csv/MD5E-s84719--c96eb5f9833f3ae071eb551a15440b9e.csv
+../../.git/annex/objects/mp/Zf/MD5E-s85190--3955ba7a965b5199502821e8b0bea5df.csv/MD5E-s85190--3955ba7a965b5199502821e8b0bea5df.csv

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/j5/Xv/MD5E-s48259--34f747d82aba0fd796f26a7d3e667cd6.csv/MD5E-s48259--34f747d82aba0fd796f26a7d3e667cd6.csv

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/mJ/q3/MD5E-s68450--6bdee1bd6dbfb88e785b30c60bc9c0e2.nc/MD5E-s68450--6bdee1bd6dbfb88e785b30c60bc9c0e2.nc

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP.yaml

@@ -0,0 +1,24 @@
+attrs:
+  references: https://unfccc.int/documents/634149
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Bangladesh. Biennial update report (BUR). BUR1 Processed on 2024-08-22 Processed
+    on 2024-08-22
+  comment: Read fom pdf by Daniel Busch Processed on 2024-08-22 Processed on 2024-08-22
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  entity: KYOTOGHG
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - source
+  - category (IPCC2006_PRIMAP)
+  - area (ISO3)
+  - provenance
+  - scenario (PRIMAP)
+  - entity
+  - unit
+data_file: BGD_BUR1_2023_IPCC2006_PRIMAP.csv

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP_raw.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/jz/0P/MD5E-s11281--6f4f7230e2e1548faebd2687a917b6c7.csv/MD5E-s11281--6f4f7230e2e1548faebd2687a917b6c7.csv

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP_raw.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/K2/Q6/MD5E-s39722--d598b22fa54ef1d596d65cd1847ac0d9.nc/MD5E-s39722--d598b22fa54ef1d596d65cd1847ac0d9.nc

extracted_data/UNFCCC/Bangladesh/BGD_BUR1_2023_IPCC2006_PRIMAP_raw.yaml

@@ -0,0 +1,22 @@
+attrs:
+  references: https://unfccc.int/documents/634149
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Bangladesh. Biennial update report (BUR). BUR1
+  comment: Read fom pdf by Daniel Busch
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - source
+  - category (IPCC2006_PRIMAP)
+  - area (ISO3)
+  - provenance
+  - scenario (PRIMAP)
+  - entity
+  - unit
+data_file: BGD_BUR1_2023_IPCC2006_PRIMAP_raw.csv

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/wk/g9/MD5E-s91654--1b95a9d247742af049b9f8a083aa26ac.csv/MD5E-s91654--1b95a9d247742af049b9f8a083aa26ac.csv

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/WG/MG/MD5E-s116273--5d8b2fdf2a5a7dcc52a25307d83c1a2e.nc/MD5E-s116273--5d8b2fdf2a5a7dcc52a25307d83c1a2e.nc

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP.yaml

@@ -0,0 +1,24 @@
+attrs:
+  references: unfccc.int/documents/638907
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Cabo Verde. Biennial update report (BUR). BUR1 Processed on 2024-08-12 Processed
+    on 2024-08-12
+  comment: Read fom pdf by Daniel Busch Processed on 2024-08-12 Processed on 2024-08-12
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  gwp_context: SARGWP100
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - source
+  - area (ISO3)
+  - provenance
+  - category (IPCC2006_PRIMAP)
+  - scenario (PRIMAP)
+  - entity
+  - unit
+data_file: CPV_BUR1_2023_IPCC2006_PRIMAP.csv

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP_raw.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/W9/5z/MD5E-s20131--304eefaee6e626f07975e16cd490aa8a.csv/MD5E-s20131--304eefaee6e626f07975e16cd490aa8a.csv

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP_raw.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/mm/Xm/MD5E-s55465--1344f4ebe111241c209c354e320de338.nc/MD5E-s55465--1344f4ebe111241c209c354e320de338.nc

extracted_data/UNFCCC/Cabo_Verde/CPV_BUR1_2023_IPCC2006_PRIMAP_raw.yaml

@@ -0,0 +1,22 @@
+attrs:
+  references: unfccc.int/documents/638907
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Cabo Verde. Biennial update report (BUR). BUR1
+  comment: Read fom pdf by Daniel Busch
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - source
+  - area (ISO3)
+  - provenance
+  - category (IPCC2006_PRIMAP)
+  - scenario (PRIMAP)
+  - entity
+  - unit
+data_file: CPV_BUR1_2023_IPCC2006_PRIMAP_raw.csv

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/9v/x2/MD5E-s351090--38627ddb9a438ef129e2dcae12446bcc.csv/MD5E-s351090--38627ddb9a438ef129e2dcae12446bcc.csv

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/F9/jk/MD5E-s278308--86ce712c212bae25a65853f7b7809828.nc/MD5E-s278308--86ce712c212bae25a65853f7b7809828.nc

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP.yaml

@@ -0,0 +1,24 @@
+attrs:
+  references: https://unfccc.int/documents/633382
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Saint Kitts and Nevis. Biennial update report (BUR). BUR1 Processed on 2024-09-09
+    Processed on 2024-09-09
+  comment: Read fom pdf by Daniel Busch Processed on 2024-09-09 Processed on 2024-09-09
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  gwp_context: AR5GWP100
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - scenario (PRIMAP)
+  - provenance
+  - area (ISO3)
+  - source
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - unit
+data_file: KNA_BUR1_2023_IPCC2006_PRIMAP.csv

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP_raw.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/Gj/fq/MD5E-s165610--0c8f9de46f98900a9ed7e6b12e17325c.csv/MD5E-s165610--0c8f9de46f98900a9ed7e6b12e17325c.csv

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP_raw.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/wk/7V/MD5E-s129950--b38f79387900334f1e9c23ed70a4e798.nc/MD5E-s129950--b38f79387900334f1e9c23ed70a4e798.nc

extracted_data/UNFCCC/Saint_Kitts_and_Nevis/KNA_BUR1_2023_IPCC2006_PRIMAP_raw.yaml

@@ -0,0 +1,22 @@
+attrs:
+  references: https://unfccc.int/documents/633382
+  rights: ''
+  contact: daniel-busch@climate-resource.de
+  title: Saint Kitts and Nevis. Biennial update report (BUR). BUR1
+  comment: Read fom pdf by Daniel Busch
+  institution: UNFCCC
+  cat: category (IPCC2006_PRIMAP)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - scenario (PRIMAP)
+  - provenance
+  - area (ISO3)
+  - source
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - unit
+data_file: KNA_BUR1_2023_IPCC2006_PRIMAP_raw.csv

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/fG/Gz/MD5E-s643827--3570eb0b519b63396e060c2d11224248.csv/MD5E-s643827--3570eb0b519b63396e060c2d11224248.csv

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/MF/Mz/MD5E-s382781--9eb80f0d624ad8f69f1fc44b8284ac98.nc/MD5E-s382781--9eb80f0d624ad8f69f1fc44b8284ac98.nc

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP.yaml

@@ -0,0 +1,25 @@
+attrs:
+  rights: ''
+  references: https://unfccc.int/documents/640099
+  contact: mail@johannes-guetschow.de
+  title: NATIONAL REPORT - Inventory of anthropogenic emissions sources and sinks
+    of greenhouse gases in the Republic of Uzbekistan 1990-2021 Processed on 2024-09-06
+  comment: Read fom pdf file by Johannes Gütschow Processed on 2024-09-06
+  institution: United Nations Framework Convention on Climate Change (UNFCCC)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  cat: category (IPCC2006_PRIMAP)
+  gwp_context: AR4GWP100
+  entity: NMVOC
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - area (ISO3)
+  - scenario (PRIMAP)
+  - source
+  - provenance
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - unit
+data_file: UZB_BTR1NIR_IPCC2006_PRIMAP.csv

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP_raw.csv

@@ -0,0 +1 @@
+../../../.git/annex/objects/2K/m2/MD5E-s287342--c961b7fef28d8685bd498591cfc631f1.csv/MD5E-s287342--c961b7fef28d8685bd498591cfc631f1.csv

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP_raw.nc

@@ -0,0 +1 @@
+../../../.git/annex/objects/7p/7W/MD5E-s186390--77608d3e9548d52c3ffb23cf622663a6.nc/MD5E-s186390--77608d3e9548d52c3ffb23cf622663a6.nc

extracted_data/UNFCCC/Uzbekistan/UZB_BTR1NIR_IPCC2006_PRIMAP_raw.yaml

@@ -0,0 +1,23 @@
+attrs:
+  rights: ''
+  references: https://unfccc.int/documents/640099
+  contact: mail@johannes-guetschow.de
+  title: NATIONAL REPORT - Inventory of anthropogenic emissions sources and sinks
+    of greenhouse gases in the Republic of Uzbekistan 1990-2021
+  comment: Read fom pdf file by Johannes Gütschow
+  institution: United Nations Framework Convention on Climate Change (UNFCCC)
+  area: area (ISO3)
+  scen: scenario (PRIMAP)
+  cat: category (IPCC2006_PRIMAP)
+time_format: '%Y'
+dimensions:
+  '*':
+  - time
+  - area (ISO3)
+  - scenario (PRIMAP)
+  - source
+  - provenance
+  - category (IPCC2006_PRIMAP)
+  - entity
+  - unit
+data_file: UZB_BTR1NIR_IPCC2006_PRIMAP_raw.csv

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
 
 [[package]]
 name = "accessible-pygments"
@@ -774,43 +774,38 @@ toml = ["tomli"]
 
 [[package]]
 name = "cryptography"
-version = "42.0.8"
+version = "43.0.1"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"},
-    {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"},
-    {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"},
-    {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"},
-    {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"},
-    {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"},
-    {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"},
-    {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"},
-    {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"},
-    {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"},
-    {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"},
-    {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"},
-    {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"},
-    {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"},
-    {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"},
-    {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"},
-    {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"},
-    {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"},
-    {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"},
-    {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"},
-    {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"},
-    {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"},
-    {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"},
-    {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"},
-    {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"},
-    {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"},
-    {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"},
-    {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"},
-    {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"},
-    {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"},
-    {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"},
-    {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"},
+    {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"},
+    {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"},
+    {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"},
+    {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"},
+    {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"},
+    {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"},
+    {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"},
+    {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"},
+    {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"},
+    {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"},
+    {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"},
+    {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"},
+    {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"},
+    {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"},
+    {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"},
+    {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"},
+    {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"},
+    {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"},
+    {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"},
+    {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"},
+    {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"},
+    {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"},
+    {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"},
+    {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"},
+    {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"},
+    {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"},
+    {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"},
 ]
 
 [package.dependencies]
@@ -823,7 +818,7 @@ nox = ["nox"]
 pep8test = ["check-sdist", "click", "mypy", "ruff"]
 sdist = ["build"]
 ssh = ["bcrypt (>=3.1.5)"]
-test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]
+test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]
 test-randomorder = ["pytest-randomly"]
 
 [[package]]
@@ -2043,13 +2038,13 @@ test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (>
 
 [[package]]
 name = "jupyterlab"
-version = "4.2.2"
+version = "4.2.5"
 description = "JupyterLab computational environment"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "jupyterlab-4.2.2-py3-none-any.whl", hash = "sha256:59ee9b839f43308c3dfd55d72d1f1a299ed42a7f91f2d1afe9c12a783f9e525f"},
-    {file = "jupyterlab-4.2.2.tar.gz", hash = "sha256:a534b6a25719a92a40d514fb133a9fe8f0d9981b0bbce5d8a5fcaa33344a3038"},
+    {file = "jupyterlab-4.2.5-py3-none-any.whl", hash = "sha256:73b6e0775d41a9fee7ee756c80f58a6bed4040869ccc21411dc559818874d321"},
+    {file = "jupyterlab-4.2.5.tar.gz", hash = "sha256:ae7f3a1b8cb88b4f55009ce79fa7c06f99d70cd63601ee4aa91815d054f46f75"},
 ]
 
 [package.dependencies]
@@ -2073,7 +2068,7 @@ dev = ["build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov",
 docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", "pytest-check-links", "pytest-jupyter", "sphinx (>=1.8,<7.3.0)", "sphinx-copybutton"]
 docs-screenshots = ["altair (==5.3.0)", "ipython (==8.16.1)", "ipywidgets (==8.1.2)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.1.post2)", "matplotlib (==3.8.3)", "nbconvert (>=7.0.0)", "pandas (==2.2.1)", "scipy (==1.12.0)", "vega-datasets (==0.9.0)"]
 test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"]
-upgrade-extension = ["copier (>=8,<10)", "jinja2-time (<0.3)", "pydantic (<2.0)", "pyyaml-include (<2.0)", "tomli-w (<2.0)"]
+upgrade-extension = ["copier (>=9,<10)", "jinja2-time (<0.3)", "pydantic (<3.0)", "pyyaml-include (<3.0)", "tomli-w (<2.0)"]
 
 [[package]]
 name = "jupyterlab-pygments"
@@ -2870,13 +2865,13 @@ files = [
 
 [[package]]
 name = "notebook"
-version = "7.2.1"
+version = "7.2.2"
 description = "Jupyter Notebook - A web-based notebook environment for interactive computing"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "notebook-7.2.1-py3-none-any.whl", hash = "sha256:f45489a3995746f2195a137e0773e2130960b51c9ac3ce257dbc2705aab3a6ca"},
-    {file = "notebook-7.2.1.tar.gz", hash = "sha256:4287b6da59740b32173d01d641f763d292f49c30e7a51b89c46ba8473126341e"},
+    {file = "notebook-7.2.2-py3-none-any.whl", hash = "sha256:c89264081f671bc02eec0ed470a627ed791b9156cad9285226b31611d3e9fe1c"},
+    {file = "notebook-7.2.2.tar.gz", hash = "sha256:2ef07d4220421623ad3fe88118d687bc0450055570cdd160814a59cf3a1c516e"},
 ]
 
 [package.dependencies]
@@ -3850,6 +3845,7 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -5247,4 +5243,4 @@ plots = ["matplotlib"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10, <3.11"
-content-hash = "3a4df7f701cbf21ac587f032fe99bcf1c73803feb6f47f135a12eec718e0cfa5"
+content-hash = "441fc47bae4e342d268715ff586e2a7f6a9ac90359a19a9edbe8086379362d09"

pyproject.toml

@@ -50,7 +50,7 @@ ruff = "^0.1.8"
 pre-commit = "^3.3.1"
 towncrier = "^23.6.0"
 liccheck = "^0.9.1"
-notebook = "^7.2.0"
+notebook = "^7.2.2"
 ipywidgets = "^8.1.2"
 ipympl = "^0.9.4"
 

src/unfccc_ghg_data/helper/__init__.py

@@ -32,6 +32,7 @@ from .functions import (
     get_code_file,
     get_country_code,
     get_country_name,
+    make_long_table,
     make_wide_table,
     process_data_for_country,
     set_to_nan_in_ds,
@@ -61,6 +62,7 @@ __all__ = [
     "get_code_file",
     "compression",
     "make_wide_table",
+    "make_long_table",
     "nAI_countries",
     "AI_countries",
     "all_countries",

src/unfccc_ghg_data/helper/functions.py

@@ -948,6 +948,8 @@ def fix_rows(
             locs_to_merge = list(range(loc - 1, loc + 1))
         elif n_rows == -3:  # noqa: PLR2004
             locs_to_merge = list(range(loc - 1, loc + 2))
+        elif n_rows == -4:  # noqa: PLR2004
+            locs_to_merge = list(range(loc - 1, loc + 3))
         elif n_rows == -5:  # noqa: PLR2004
             locs_to_merge = list(range(loc - 1, loc + 4))
         else:
@@ -1033,6 +1035,68 @@ def make_wide_table(
         return df_all
 
 
+def make_long_table(
+    data: pd.DataFrame,
+    keyword: str,
+) -> pd.DataFrame | None:
+    """
+    Make long table from a table horizontally combined blocks
+
+    Make a long table from a table which is a horizontal
+    concatenation of tables for different time periods
+
+    This function assumes that the table header is in
+    the first row and the columns are numerical
+
+    Parameters
+    ----------
+    data
+        Input table as pandas.DataFrame
+    keyword
+
+    Returns
+    -------
+    pandas.DataFrame in long format
+
+    """
+    kw_locs = data.iloc[0] == keyword
+    kw_locs = list(np.where(kw_locs)[0])
+
+    if len(kw_locs) < 2:  # noqa: PLR2004
+        print(f'Keyword "{keyword}" not found multiple times')
+        return data
+
+    else:
+        df_all = None
+        for ikw in range(len(kw_locs)):
+            if ikw < len(kw_locs) - 1:
+                end_pos = kw_locs[ikw + 1]
+            else:
+                end_pos = len(data.columns)
+            df_to_add = data.iloc[:, range(kw_locs[ikw], end_pos)]
+            df_to_add.columns = df_to_add.iloc[0]
+            idx_to_drop = df_to_add.index[0]
+            df_to_add = df_to_add.drop(idx_to_drop)
+
+            # remove all rows which have no year
+            filter_nan = (
+                (~df_to_add[keyword].isna())
+                & (df_to_add[keyword] != "NaN")
+                & (df_to_add[keyword] != "")
+            )
+            df_to_add = df_to_add.loc[filter_nan]
+
+            if df_all is None:
+                df_all = df_to_add
+            else:
+                try:
+                    df_all = pd.concat([df_all, df_to_add])
+                except Exception as ex:
+                    print(f"Table parts not compatible, Exception: {ex}")
+
+        return df_all
+
+
 def find_and_replace_values(
     df: pd.DataFrame,
     replace_info: list[tuple[str | float]],

src/unfccc_ghg_data/unfccc_reader/Bangladesh/__init__.py

@@ -0,0 +1,30 @@
+"""Read Bangladesh's BURs, NIRs, NCs
+
+Scripts and configurations to read Burundi's submissions to the UNFCCC.
+Currently, the following submissions and datasets are available (all datasets
+including DI (red using the DI-reader) and legacy BUR/NIR (no code)):
+
+.. exec_code::
+    :hide_code:
+
+    from unfccc_ghg_data.helper.functions import (get_country_datasets,
+                                                  get_country_submissions)
+    country = 'BGD'
+    # print available submissions
+    print("="*15 + " Available submissions " + "="*15)
+    get_country_submissions(country, True)
+    print("")
+
+    #print available datasets
+    print("="*15 + " Available datasets " + "="*15)
+    get_country_datasets(country, True)
+
+You can also obtain this information running
+
+.. code-block:: bash
+
+    poetry run doit country_info country=BGD
+
+See below for a listing of scripts for BUR/NIR reading including links.
+
+"""

src/unfccc_ghg_data/unfccc_reader/Bangladesh/config_bgd_bur1.py

@@ -0,0 +1,1053 @@
+"""
+Configuration file to read Bangladesh's BUR 1.
+
+# Overview of all available GHG tables
+
+# not reading:
+# table 7, page 70 - already in main table in annex
+# table 8, page 71 - only four new data points for 2012
+# figure 22, page 83 - this is a summary of all energy tables
+# figure 23 - image of summary of IPUU, data are available as tables in same
+# chapter
+# table 27 - rice cultivation available - in main tables
+# table 28 - N2O from fertilizers - in main tables
+# table 29 - indirect N2O from fertilizer - in main table
+# table 31 - enteric CH4 by livestock - low priority
+# table 32 - manure CH4 by livestock - low priority
+# table 37 - already in main tables in annex
+# table 19-23, ammonia-urea, cement, glass, lubricants, steel mills
+# 2013-2019 on pages 86-88 - already in main table
+
+# reading:
+# table 16, page 78 - 2013-2019 by industry sub-sectors - typed
+# table 17, page 79 - 2013-2019 transportation - typed
+# table 18, page 80 - residential 2013-2019 commercial sector  - typed
+# table 19, page 80 - agriculture energy use 2013-2019 - typed
+# figure 20, page 81 - gas leakage 2013-2019 - typed
+
+
+"""
+
+coords_terminologies = {
+    "area": "ISO3",
+    "category": "IPCC2006_PRIMAP",
+    "scenario": "PRIMAP",
+}
+
+# primap2 format conversion
+coords_cols = {
+    "category": "category",
+    "entity": "entity",
+    "unit": "unit",
+}
+
+coords_defaults = {
+    "source": "BGD-GHG-Inventory",
+    "provenance": "measured",
+    "area": "BGD",
+    "scenario": "BUR1",
+}
+
+gwp_to_use = "AR4GWP100"
+
+coords_value_mapping = {
+    "unit": "PRIMAP1",
+    "category": "PRIMAP1",
+}
+
+filter_remove = {
+    "f_memo": {"category": "MEMO"},
+    "f_info": {"category": "INFO"},
+}
+
+meta_data = {
+    "references": "https://unfccc.int/documents/634149",
+    "rights": "",  # unknown
+    "contact": "daniel-busch@climate-resource.de",
+    "title": "Bangladesh. Biennial update report (BUR). BUR1",
+    "comment": "Read fom pdf by Daniel Busch",
+    "institution": "UNFCCC",
+}
+
+inv_conf = {
+    "entity_row": 0,
+    "unit_row": 1,
+    "index_cols": "Greenhouse gas source and sink categories",
+    "header_long": ["orig_cat_name", "entity", "unit", "time", "data"],
+    "cat_code_regexp": r"^(?P<code>[a-zA-Z0-9\.]{1,11})[\s\.].*",
+}
+inv_conf_per_year = {
+    "2013": {
+        "page_defs": {
+            "207": {
+                "area": ["60,630,534,79"],
+                "cols": ["387,444,495"],
+                "skip_rows_start": 0,
+                "skip_rows_end": 0,
+            },
+            "208": {
+                "area": ["63,720,527,120"],
+                "cols": ["380,437,491"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: [
+                "3 - GHG Emissions Agriculture, Livestock & Forest and Other Land -Use"
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 6,
+        # TODO The manual codes can be summarised for all years
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+    },
+    "2014": {
+        "page_defs": {
+            "209": {
+                "area": ["74,715,542,78"],
+                "cols": ["380,441,498"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "210": {
+                "area": ["64,715,529,196"],
+                "cols": ["380,435,491"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-a - Fuel combustion activities_energy industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+    },
+    "2015": {
+        "page_defs": {
+            "211": {
+                "area": ["75,712,550,88"],
+                "cols": ["375,444,498"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "212": {
+                "area": ["64,711,524,90"],
+                "cols": ["369,436,492"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+                "a-co2 emission from soil",
+                "c-co2 emission due to fuel wood removal for consumption",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste wate"r" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+        # These three categories only appear in 2015 and are all zero
+        "categories_to_drop": [
+            "a-co2 emission from soil",
+            "c-co2 emission due to fuel wood removal for consumption",
+            "B-conversion of forest land to other land use",
+        ],
+    },
+    "2016": {
+        "page_defs": {
+            "213": {
+                "area": ["73,712,544,77"],
+                "cols": ["373,444,498"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "214": {
+                "area": ["66,711,533,143"],
+                "cols": ["359,435,492"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+    },
+    "2017": {
+        "page_defs": {
+            "215": {
+                "area": ["74,715,543,80"],
+                "cols": ["382,444,497"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "216": {
+                "area": ["64,720,530,158"],
+                "cols": ["380,433,490"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "2 a. 1-cement Production": "2.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+    },
+    "2018": {
+        "page_defs": {
+            "217": {
+                "area": ["75,713,542,91"],
+                "cols": ["378,446,499"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "218": {
+                "area": ["63,714,528,154"],
+                "cols": ["374,438,491"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo items (5)": "MEMO",
+            "Information Items": "INFO",
+            "international Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+    },
+    "2019": {
+        "page_defs": {
+            "219": {
+                "area": ["75,713,542,91"],
+                "cols": ["378,446,499"],
+                "skip_rows_start": 9,
+                "skip_rows_end": 0,
+            },
+            "220": {
+                "area": ["63,714,524,139"],
+                "cols": ["374,438,491"],
+                "skip_rows_start": 8,
+                "skip_rows_end": 4,
+            },
+        },
+        "rows_to_fix": {
+            3: ["3 - GHG Emissions Agriculture, Livestock & Forest and Other"],
+            -2: [
+                "ch4 emission from rice field",
+                "indirect nitrous oxide (n2o) from n based fertilizer",
+                "Direct nitrous oxide (n2o) emissions from fertilizer application",
+                "Total enteric ch4 emissions",
+                "Total Manure ch4 emissions",
+                "Total Direct n2o emissions from manure system",
+                "Total indirect n2o emissions - Volatilization",
+                "Total indirect n2o emissions - leaching/Runoff",
+                "CO2 from Biomass burning for Energy purpose",
+            ],
+            2: [
+                # "B-Methane emission from domestic waste water" and
+                # "c-nitrous oxide emission from domestic waste water" are the same category
+                # and should be merged
+                "B-Methane emission from domestic waste water",
+                # "Total Manure ch4 emissions" and
+                # "Total Direct n2o emissions from manure system" are the same category
+                # and should be merged
+                "Total Manure ch4 emissions",
+            ],
+        },
+        "header": [
+            "Greenhouse gas source and sink categories",
+            "CO2",
+            "CH4",
+            "N2O",
+        ],
+        "unit": [
+            "-",
+            "Gg",
+            "Gg",
+            "Gg",
+        ],
+        "skip_rows": 0,
+        "cat_codes_manual": {
+            "1-A - Fuel Combustion Activities_Energy Industries": "1.A",
+            "1 - a1- electricity Generation": "1.A.1",
+            "1.a2- Manufacturing industries and construction": "1.A.2",
+            "1.a3-Transport": "1.A.3",
+            "1.a4-other sectors": "1.A.4",
+            "2 a. 1-cement Production": "2.A.1",
+            "2a3 Glass Production": "2.A.3",
+            "2 B. 1 - ammonia Production": "2.B.1",
+            "2 C-Metal Industry": "2.C",
+            "2 c. 1 iron and steel Production": "2.C.1",
+            "2. D - Non-Energy Products from Fuels and Solvent Use": "2.D",
+            "2D 1-lubricant Use": "2.D.1",
+            "ch4 emission from rice field": "3.C.7",
+            "indirect nitrous oxide (n2o) from n based fertilizer": "3.C.5",
+            "Direct nitrous oxide (n2o) emissions from fertilizer application": "3.C.4",
+            "Direct carbon Dioxide emissions from urea fertilizer": "3.C.3",
+            "Total enteric ch4 emissions": "3.A.1",
+            "Total Manure ch4 emissions Total Direct n2o emissions from manure system": "3.A.2",
+            "Total indirect n2o emissions -Volatilization": "3.C.5",
+            "Total indirect n2o emissions -leaching/Runoff": "3.C.5",
+            "4 a-solid Waste Disposal": "4.A",
+            "B-Methane emission from domestic waste water c-nitrous oxide emission from domestic waste water": "4.D.1",
+            "D- Metahne emission from industrial waste water": "4.D.2",
+            "Memo Items (5)": "MEMO",
+            "Information Items": "INFO",
+            "International Bunkers": "M.BK",
+            "a-international aviation (international Bunkers)": "M.BK.A",
+            "B-international Water-borne navigation (international Bunkers)": "M.BK.M",
+            "CO2 from Biomass burning for Energy purpose": "M.BIO",
+        },
+        "merge_cats": "3C5",
+        "categories_to_drop": ["in eq. Million Tons"],
+    },
+}
+
+# needed for the pandas wide_to_long function
+wide_to_long_col_replace = {
+    "2013": "data2013",
+    "2014": "data2014",
+    "2015": "data2015",
+    "2016": "data2016",
+    "2017": "data2017",
+    "2018": "data2018",
+    "2019": "data2019",
+}
+
+manually_typed = {
+    "figure_16": {
+        # In other than stated in the figure, these are KYOTO gases in CO2eq
+        "unit": "GgCO2eq",
+        "entity": "KYOTOGHG (AR4GWP100)",
+        "data": {
+            "category": [
+                "1.A.2.a",
+                "1.A.2.b",
+                "1.A.2.c",
+                "1.A.2.d",
+                "1.A.2.e",
+                "1.A.2.f",
+                "1.A.2.g",
+                "1.A.2.h",
+                "1.A.2.i",
+                "1.A.2.j",
+                "1.A.2.k",
+                "1.A.2.l",
+                "1.A.2.m",
+            ],
+            "2013": [
+                709,
+                6,
+                796,
+                421,
+                553,
+                12174,
+                1,
+                1,
+                0,
+                4,
+                5,
+                4885,
+                1280,
+            ],
+            "2014": [
+                706,
+                0,
+                636,
+                409,
+                515,
+                13896,
+                0,
+                0,
+                0,
+                0,
+                203,
+                4793,
+                1276,
+            ],
+            "2015": [
+                778,
+                0,
+                545,
+                458,
+                532,
+                13660,
+                0,
+                0,
+                0,
+                0,
+                195,
+                5180,
+                1799,
+            ],
+            "2016": [
+                830,
+                0,
+                445,
+                473,
+                527,
+                15771,
+                0,
+                0,
+                0,
+                0,
+                118,
+                5375,
+                1700,
+            ],
+            "2017": [
+                883,
+                0,
+                344,
+                492,
+                522,
+                15141,
+                0,
+                0,
+                0,
+                0,
+                179,
+                5294,
+                1546,
+            ],
+            "2018": [
+                943,
+                0,
+                247,
+                519,
+                519,
+                15949,
+                0,
+                0,
+                0,
+                0,
+                193,
+                5810,
+                1764,
+            ],
+            "2019": [
+                988,
+                0,
+                123,
+                527,
+                516,
+                16091,
+                0,
+                0,
+                0,
+                0,
+                216,
+                5935,
+                2492,
+            ],
+        },
+    },
+    "figure_17": {
+        # In other than stated in the figure, these are KYOTO gases in CO2eq
+        "unit": "GgCO2eq",
+        "entity": "KYOTOGHG (AR4GWP100)",
+        "data": {
+            "category": [
+                "1.A.3.a.ii",
+                "1.A.3.b.i.2",
+                "1.A.3.b.ii.2",
+                "1.A.3.b.iii",
+                "1.A.3.b.iv",
+                "1.A.3.c",
+                "1.A.3.d.ii",
+            ],
+            "2013": [
+                694,
+                1450,
+                1215,
+                8960,
+                979,
+                115,
+                162,
+            ],
+            "2014": [
+                704,
+                1485,
+                914,
+                8126,
+                934,
+                113,
+                196,
+            ],
+            "2015": [
+                738,
+                1708,
+                1135,
+                9082,
+                1030,
+                117,
+                208,
+            ],
+            "2016": [
+                757,
+                1710,
+                1030,
+                8504,
+                1089,
+                115,
+                350,
+            ],
+            "2017": [
+                822,
+                1962,
+                1298,
+                10201,
+                1136,
+                156,
+                289,
+            ],
+            "2018": [
+                890,
+                2019,
+                1410,
+                10320,
+                1152,
+                140,
+                332,
+            ],
+            "2019": [
+                938,
+                2440,
+                1985,
+                12682,
+                1232,
+                168,
+                401,
+            ],
+        },
+    },
+    "figure_18": {
+        # In other than stated in the figure, these are KYOTO gases in CO2eq
+        "unit": "GgCO2eq",
+        "entity": "KYOTOGHG (AR4GWP100)",
+        "data": {
+            "category": ["1.A.4.a", "1.A.4.b"],
+            "2013": [1871, 6703],
+            "2014": [1619, 6960],
+            "2015": [1522, 8573],
+            "2016": [1260, 9755],
+            "2017": [981, 9702],
+            "2018": [833, 11355],
+            "2019": [835, 12317],
+        },
+    },
+    "figure_19": {
+        # In other than stated in the figure, these are KYOTO gases in CO2eq
+        "unit": "GgCO2eq",
+        "entity": "KYOTOGHG (AR4GWP100)",
+        "data": {
+            "category": ["1.A.4.c.i", "1.A.4.c.iii", "1.A.4.c"],
+            "2013": [2692, 5, 2697],
+            "2014": [2804, 5, 2809],
+            "2015": [2977, 6, 2983],
+            "2016": [3035, 6, 3040],
+            "2017": [2903, 6, 2909],
+            "2018": [3496, 6, 3502],
+            "2019": [3446, 6, 3452],
+        },
+    },
+    "figure_20": {
+        "unit": "GgCO2eq",
+        "entity": "CH4",
+        "data": {
+            "category": ["1.B.2.b.iii.4", "1.B.2.b.iii.5", "1.B.2.b.iii"],
+            "2013": [896, 8440, 9336],
+            "2014": [896, 8440, 9336],
+            "2015": [896, 8440, 9336],
+            "2016": [896, 8440, 9336],
+            "2017": [896, 6429, 7325],
+            "2018": [896, 6429, 7325],
+            "2019": [896, 4289, 5185],
+        },
+        "unit_conversion": {"new_unit": "Gg", "conversion_factor": 25},
+    },
+}
+
+# correct values that are obviously wrong in the tables
+values_to_correct = [
+    # the sum of 1.A sub-categories does not match the value of 1.A
+    ("1.A", "CH4", "2014", 110),
+    ("1.A", "CO2", "2014", 77373),
+    ("1", "N2O", "2014", 3.8),
+    # For the sum for CO2 in category 3 they forgot to add 3.B
+    ("3", "CO2", "2013", 8140),
+    ("3", "CO2", "2014", 8923),
+    ("3", "CO2", "2015", 9791),
+    ("3", "CO2", "2016", 10518),
+    ("3", "CO2", "2017", 11359),
+    ("3", "CO2", "2018", 11993),
+    ("3", "CO2", "2019", 12640),
+]
+
+country_processing_step1 = {
+    "tolerance": 0.01,
+    "aggregate_cats": {
+        "3.A": {"sources": ["3.A.1", "3.A.2"]},
+        "M.3.C.AG": {  # "Aggregate sources and non-CO2 emissions sources on land (Agriculture)"
+            "sources": [
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.7",
+            ]
+        },
+        "3.C": {"sources": ["3.C.3", "3.C.4", "3.C.5", "3.C.7"]},
+        "M.AG.ELV": {
+            "sources": ["M.3.C.AG", "M.3.D.AG"],
+        },
+        "M.AG": {"sources": ["3.A", "M.AG.ELV"]},  # agriculture
+        "M.LULUCF": {"sources": ["3.B", "M.3.D.LU"]},
+        "M.0.EL": {
+            "sources": ["1", "2", "M.AG", "4"],
+        },
+        "1.B": {"sources": ["1.B.2"]},
+        "4.D": {"sources": ["4.D.1", "4.D.2"]},
+        "1": {"sources": ["1.A", "1.B"]},  # consistency check energy
+        "2": {"sources": ["2.A", "2.B", "2.C", "2.D", "2.F"]},  # consistency check IPPU
+        "3": {"sources": ["M.AG", "M.LULUCF"]},  # consistency check AFOLU
+        "4": {"sources": ["4.A", "4.D"]},  # consistency check waste
+        # check if typed numbers add up to the total of 1.A.2 from the main table
+        "1.A.2": {
+            "sources": [
+                "1.A.2.a",
+                "1.A.2.b",
+                "1.A.2.c",
+                "1.A.2.d",
+                "1.A.2.e",
+                "1.A.2.f",
+                "1.A.2.g",
+                "1.A.2.h",
+                "1.A.2.i",
+                "1.A.2.j",
+                "1.A.2.k",
+                "1.A.2.l",
+                "1.A.2.m",
+            ]
+        },
+        # check if typed numbers add up to the total of 1.A.3 from the main table
+        "1.A.3": {
+            "sources": [
+                "1.A.3.a.ii",
+                "1.A.3.b.i.2",
+                "1.A.3.b.ii.2",
+                "1.A.3.b.iii",
+                "1.A.3.b.iv",
+                "1.A.3.c",
+                "1.A.3.d.ii",
+            ]
+        },
+        # check if the typed numbers add up to the total of 1.A.4.c in the same table
+        "1.A.4.c": {"sources": ["1.A.4.c.i", "1.A.4.c.iii"]},
+        # check if typed numbers add up to the total of 1.A.4 from the main table
+        "1.A.4": {"sources": ["1.A.4.a", "1.A.4.b", "1.A.4.c"]},
+        # check if the typed numbers add up to the total of 1.A.4.c in the same table
+        "1.B.2.b.iii": {"sources": ["1.B.2.b.iii.4", "1.B.2.b.iii.5"]},
+        # consistency check for 1.B.2
+        "1.B.2": {"sources": ["1.B.2.b.iii"]},
+    },
+    # We don't have HFCs and PFCs in the report, hence basket_copy is not relevant
+    # "basket_copy": {
+    #     "GWPs_to_add": ["SARGWP100", "AR5GWP100", "AR6GWP100"],
+    #     # "entities": ["HFCS", "PFCS"],
+    #     "source_GWP": gwp_to_use,
+    # },
+}
+
+# Note on downscaling: Data are always available for the same years: 2013-2019,
+# so temporal downscaling does not makes sense here.
+# TODO: Perhaps entity, category downscaling can be done?
+
+gas_baskets = {
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O"],
+}

src/unfccc_ghg_data/unfccc_reader/Bangladesh/read_BGD_BUR1_from_pdf.py

@@ -0,0 +1,340 @@
+"""
+Read Bangladesh's BUR1 from pdf
+"""
+
+import camelot
+import numpy as np
+import pandas as pd
+import primap2 as pm2
+
+from unfccc_ghg_data.helper import (
+    downloaded_data_path,
+    extracted_data_path,
+    fix_rows,
+    process_data_for_country,
+)
+from unfccc_ghg_data.unfccc_reader.Bangladesh.config_bgd_bur1 import (
+    coords_cols,
+    coords_defaults,
+    coords_terminologies,
+    coords_value_mapping,
+    country_processing_step1,
+    filter_remove,
+    gas_baskets,
+    inv_conf,
+    inv_conf_per_year,
+    manually_typed,
+    meta_data,
+    values_to_correct,
+    wide_to_long_col_replace,
+)
+
+if __name__ == "__main__":
+    # ###
+    # configuration
+    # ###
+
+    input_folder = downloaded_data_path / "UNFCCC" / "Bangladesh" / "BUR1"
+    output_folder = extracted_data_path / "UNFCCC" / "Bangladesh"
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+
+    pdf_file = "Updated_BUR1_Report_15_11_2023.pdf"
+    output_filename = "BGD_BUR1_2023_"
+    category_column = f"category ({coords_terminologies['category']})"
+    compression = dict(zlib=True, complevel=9)
+
+    def repl(m):  # noqa: D103
+        return m.group("code")
+
+    # ###
+    # 1. Read in main tables from the Annex
+    # ###
+    df_main = None
+    df_year = None
+    for year in reversed(list(inv_conf_per_year.keys())):
+        print("-" * 60)
+        print(f"Reading year {year}.")
+        print("-" * 60)
+        df_year = None
+        for page in inv_conf_per_year[year]["page_defs"].keys():
+            print(f"Reading table from page {page}.")
+
+            # read from PDF
+            tables_inventory_original = camelot.read_pdf(
+                str(input_folder / pdf_file),
+                pages=page,
+                table_areas=inv_conf_per_year[year]["page_defs"][page]["area"],
+                columns=inv_conf_per_year[year]["page_defs"][page]["cols"],
+                flavor="stream",
+                split_text=False,
+            )
+            print("Reading complete.")
+
+            df_page = tables_inventory_original[0].df
+
+            # cut rows at the top if needed
+            skip_rows_start = inv_conf_per_year[year]["page_defs"][page][
+                "skip_rows_start"
+            ]
+            if not skip_rows_start == 0:
+                df_page = df_page[skip_rows_start:]
+
+            # cut rows at the bottom if needed
+            skip_rows_end = inv_conf_per_year[year]["page_defs"][page]["skip_rows_end"]
+            if not skip_rows_end == 0:
+                df_page = df_page[:-skip_rows_end]
+
+            # stack the tables vertically
+            if df_year is None:
+                df_year = df_page
+            else:
+                df_year = pd.concat(
+                    [
+                        df_year,
+                        df_page,
+                    ],
+                    axis=0,
+                    join="outer",
+                ).reset_index(drop=True)
+
+        # fix content that spreads across multiple rows
+        if "rows_to_fix" in inv_conf_per_year[year]:
+            for n_rows in inv_conf_per_year[year]["rows_to_fix"].keys():
+                print(f"Merge content for {n_rows=}")
+                df_year = fix_rows(
+                    df_year,
+                    rows_to_fix=inv_conf_per_year[year]["rows_to_fix"][n_rows],
+                    col_to_use=0,
+                    n_rows=n_rows,
+                )
+
+        if "categories_to_drop" in inv_conf_per_year[year]:
+            for row in inv_conf_per_year[year]["categories_to_drop"]:
+                row_to_delete = df_year.index[df_year[0] == row][0]
+                df_year = df_year.drop(index=row_to_delete)
+
+        df_header = pd.DataFrame(
+            [inv_conf_per_year[year]["header"], inv_conf_per_year[year]["unit"]]
+        )
+        skip_rows = inv_conf_per_year[year]["skip_rows"]
+
+        df_year = pd.concat(
+            [df_header, df_year[skip_rows:]], axis=0, join="outer"
+        ).reset_index(drop=True)
+
+        df_year = pm2.pm2io.nir_add_unit_information(
+            df_year,
+            unit_row=1,
+            entity_row=0,
+            regexp_entity=".*",
+            regexp_unit=".*",
+            default_unit="Gg",
+        )
+
+        print("Added unit information.")
+
+        # set index
+        df_year = df_year.set_index(inv_conf["index_cols"])
+
+        # convert to long format
+        df_year_long = pm2.pm2io.nir_convert_df_to_long(
+            df_year, year, inv_conf["header_long"]
+        )
+
+        # extract from tuple
+        df_year_long["orig_cat_name"] = df_year_long["orig_cat_name"].str[0]
+
+        # prep for conversion to PM2 IF and native format
+        # make a copy of the categories row
+        df_year_long["category"] = df_year_long["orig_cat_name"]
+
+        # first the manual replacements
+        df_year_long["category"] = df_year_long["category"].replace(
+            inv_conf_per_year[year]["cat_codes_manual"]
+        )
+
+        # Remove dots between letters in category codes
+        df_year_long["category"] = df_year_long["category"].str.replace(".", "")
+        # Some categories have a dash between the letters
+        df_year_long["category"] = df_year_long["category"].str.replace("-", " ")
+
+        # then the regex replacements
+        df_year_long["category"] = df_year_long["category"].str.replace(
+            inv_conf["cat_code_regexp"], repl, regex=True
+        )
+
+        df_year_long = df_year_long.reset_index(drop=True)
+
+        # make sure all col headers are str
+        df_year_long.columns = df_year_long.columns.map(str)
+
+        df_year_long = df_year_long.drop(columns=["orig_cat_name"])
+
+        # TODO Is there a better way to do this?
+        # merge duplicate categories and sum their values
+        if "merge_cats" in inv_conf_per_year[year]:
+            cat = inv_conf_per_year[year]["merge_cats"]
+            # filter by category to be merged
+            df_temp = df_year_long.loc[df_year_long["category"] == cat]
+            df_temp = df_temp.replace("", np.nan)
+            df_temp["data"] = df_temp["data"].apply(float)
+            # sum values for duplicate entries
+            df_temp = df_temp.groupby(
+                ["entity", "unit", "time", "category"], as_index=False
+            )["data"].sum()
+            # change back to empty strings
+            df_temp = df_temp.replace(0, "")
+            # drop category from df
+            df_year_long = df_year_long.drop(
+                df_year_long[df_year_long["category"] == cat].index
+            )
+            # append the summed up sub-set
+            df_year_long = pd.concat(
+                [df_temp, df_year_long],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+        if df_main is None:
+            df_main = df_year_long
+        else:
+            df_main = pd.concat(
+                [df_main, df_year_long],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    # ###
+    # 2. Add manually typed tables
+    # ###
+
+    df_typed = None
+    for figure in manually_typed.keys():
+        df_typed_figure = pd.DataFrame(manually_typed[figure]["data"])
+        df_typed_figure["entity"] = manually_typed[figure]["entity"]
+        df_typed_figure["unit"] = manually_typed[figure]["unit"]
+
+        # adjust column names for wide to long function
+        df_typed_figure = df_typed_figure.rename(columns=wide_to_long_col_replace)
+        df_typed_figure_long = pd.wide_to_long(
+            df_typed_figure, stubnames="data", i="category", j="time"
+        ).reset_index()
+
+        if "unit_conversion" in manually_typed[figure].keys():
+            df_typed_figure_long["unit"] = manually_typed[figure]["unit_conversion"][
+                "new_unit"
+            ]
+            conv_factor = manually_typed[figure]["unit_conversion"]["conversion_factor"]
+            df_typed_figure_long["data"] = df_typed_figure_long["data"].map(
+                lambda a: a / conv_factor
+            )
+
+        if df_typed is None:
+            df_typed = df_typed_figure_long
+        else:
+            df_typed = pd.concat(
+                [df_typed, df_typed_figure_long],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    # merge manually typed and main tables from Annex
+    df_main = pd.concat(
+        [df_main, df_typed],
+        axis=0,
+        join="outer",
+    ).reset_index(drop=True)
+
+    ### convert to interchange format ###
+    print("Converting to interchange format.")
+    df_main_IF = pm2.pm2io.convert_long_dataframe_if(
+        df_main,
+        coords_cols=coords_cols,
+        coords_defaults=coords_defaults,
+        coords_terminologies=coords_terminologies,
+        coords_value_mapping=coords_value_mapping,
+        filter_remove=filter_remove,
+        meta_data=meta_data,
+        convert_str=True,
+        time_format="%Y",
+    )
+
+    # correct individual values
+    for category, entity, year, new_value in values_to_correct:
+        df_main_IF.loc[
+            (df_main_IF["category (IPCC2006_PRIMAP)"] == category)
+            & (df_main_IF["entity"] == entity),
+            year,
+        ] = new_value
+
+    ### convert to primap2 format ###
+    print("Converting to primap2 format.")
+    data_pm2 = pm2.pm2io.from_interchange_format(df_main_IF)
+
+    # # ###
+    # # Save raw data to IF and native format.
+    # # ###
+
+    data_if = data_pm2.pr.to_interchange_format()
+
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + coords_terminologies["category"] + "_raw"),
+        data_if,
+    )
+
+    encoding = {var: compression for var in data_pm2.data_vars}
+    data_pm2.pr.to_netcdf(
+        output_folder
+        / (output_filename + coords_terminologies["category"] + "_raw.nc"),
+        encoding=encoding,
+    )
+
+    # # ###
+    # # Processing
+    # # ###
+
+    # create the gas baskets before aggregating the categories
+    data_proc_pm2_gas_baskets = process_data_for_country(
+        data_country=data_pm2,
+        entities_to_ignore=[],
+        gas_baskets=gas_baskets,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=None,
+    )
+
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_proc_pm2_gas_baskets,
+        entities_to_ignore=[],
+        gas_baskets=None,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step1,
+    )
+
+    # # ###
+    # # save processed data to IF and native format
+    # # ###
+
+    terminology_proc = coords_terminologies["category"]
+
+    data_proc_if = data_proc_pm2.pr.to_interchange_format()
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + terminology_proc), data_proc_if
+    )
+
+    encoding = {var: compression for var in data_proc_pm2.data_vars}
+    data_proc_pm2.pr.to_netcdf(
+        output_folder / (output_filename + terminology_proc + ".nc"), encoding=encoding
+    )
+
+    print("Saved processed data.")

src/unfccc_ghg_data/unfccc_reader/Cabo_Verde/__init__.py

@@ -0,0 +1,30 @@
+"""Read Cabo Verde's BURs, NIRs, NCs
+
+Scripts and configurations to read Cabo Verde's submissions to the UNFCCC.
+Currently, the following submissions and datasets are available (all datasets
+including DI (red using the DI-reader) and legacy BUR/NIR (no code)):
+
+.. exec_code::
+    :hide_code:
+
+    from unfccc_ghg_data.helper.functions import (get_country_datasets,
+                                                  get_country_submissions)
+    country = 'CPV'
+    # print available submissions
+    print("="*15 + " Available submissions " + "="*15)
+    get_country_submissions(country, True)
+    print("")
+
+    #print available datasets
+    print("="*15 + " Available datasets " + "="*15)
+    get_country_datasets(country, True)
+
+You can also obtain this information running
+
+.. code-block:: bash
+
+    poetry run doit country_info country=CPV
+
+See below for a listing of scripts for BUR/NIR reading including links.
+
+"""

src/unfccc_ghg_data/unfccc_reader/Cabo_Verde/config_cpv_bur1.py

@@ -0,0 +1,374 @@
+"""
+Configuration for Cabo Verde BUR1 (read from pdf)
+"""
+
+# reading tables on pages:
+# 33/1, GHG emissions and removals by type of gas, by sector and by year
+# 39, Total GHG Emissions, in CO2eq, for international bunkers, in 1995, 2000, 2005, 2010, 2015 and 2019
+# 86-89, GHG emissions in 2019
+# Not reading tables on pages:
+# 37/38, has additional columns on PFCs, Unspecified mixture of HFCs and PFCs,
+# and SF6, but they are all empty
+# 32, same information as in table 33/1
+# 33/2, aggregation of table 33/1
+# 43, no new information here
+
+coords_terminologies = {
+    "area": "ISO3",
+    "category": "IPCC2006_PRIMAP",
+    "scenario": "PRIMAP",
+}
+
+# primap2 format conversion
+coords_cols = {
+    "category": "category",
+    "entity": "entity",
+    "unit": "unit",
+}
+
+coords_defaults = {
+    "source": "CPV-GHG-Inventory",
+    "provenance": "measured",
+    "area": "CPV",
+    "scenario": "BUR1",
+}
+
+gwp_to_use = "SARGWP100"
+
+coords_value_mapping_main = {
+    "unit": "PRIMAP1",
+    "category": "PRIMAP1",
+    "entity": {
+        "HFCs": f"HFCS ({gwp_to_use})",
+        "HFC": f"HFCS ({gwp_to_use})",
+    },
+}
+
+coords_value_mapping = {
+    "unit": "PRIMAP1",
+    "category": "PRIMAP1",
+    "entity": {
+        "CO²": "CO2",
+        "CH⁴": "CH4",
+        "N²O": "N2O",
+        "F-gases": f"FGASES ({gwp_to_use})",
+    },
+}
+
+filter_remove = {
+    "f_memo": {"category": "MEMO"},
+    # They are all NaN and don't match a pre-defined entity
+    "f_fluor": {"entity": "Other fluorinated products"},
+}
+
+meta_data = {
+    "references": "unfccc.int/documents/638907",
+    "rights": "",  # unknown
+    "contact": "daniel-busch@climate-resource.de",
+    "title": "Cabo Verde. Biennial update report (BUR). BUR1",
+    "comment": "Read fom pdf by Daniel Busch",
+    "institution": "UNFCCC",
+}
+
+trend_years = ["1995", "2000", "2005", "2010", "2015", "2019"]
+
+inv_conf_per_sector = {
+    "main": {
+        "page": "33",
+        "skip_rows_start": 2,
+        "cat_codes_manual": {
+            "Energy": "1",
+            "IPPU": "2",
+            "Agriculture": "M.AG",
+            "LULUCF": "M.LULUCF",
+            "Waste": "4",
+        },
+        "header": ["category", "entity", *trend_years],
+        "unit": ["Gg"] * 4 + ["GgCO2eq"] + ["Gg"] * 9,
+        "unit_conversion": {
+            "index": 6,
+            "conversion_factor": 2240.625,
+        },
+        # "unit": ["Gg"] * 4 + ["Gg CO2eq"] + ["Gg"] * 9,
+    },
+    "int_bunkers": {
+        "page": "39",
+        "skip_rows_start": 2,
+        "cat_codes_manual": {
+            "Total International Bunkers": "M.BK",
+            "International aviation": "M.BK.A",
+            "International shipping": "M.BK.M",
+        },
+        "header": ["category", *trend_years],
+        "unit": "Gg CO2eq",
+        "drop_cols": 7,
+        "entity": "KYOTOGHG (SARGWP100)",
+    },
+}
+
+inv_conf = {
+    "cat_code_regexp": r"^(?P<code>[a-zA-Z0-9\.]{1,11})[\s\.].*",
+    "year": "2019",
+    # TODO check again!
+    # "CO2 emissions from Biomass" and "CO2 emissions from using manure as energy" are the same category
+    "merge_cats": "MBIO",
+}
+
+inv_conf_main = {
+    "pages": {
+        "86": {
+            "skip_rows_start": 2,
+            "entities": ["CO2", "CH4", "N2O"],
+            "column_names": ["category", "CO2", "CH4", "N2O"],
+            "cat_codes_manual": {
+                "Memo items": "MEMO",
+                "International bunkers": "M.BK",
+                "CO² emissions from Biomass": "M.BIO",
+                "CO² emissions from using manure as energy": "M.BIO",
+            },
+            "unit_for_entity": {
+                "CO2": "Gg",
+                "CH4": "Gg",
+                "N2O": "Gg",
+            },
+            # "units" : ["no unit", "Gg", "Gg", "Gg"]
+        },
+        "87": {
+            "skip_rows_start": 2,
+            "entities": ["CO2", "CH4", "N2O", "HFCs", "Other fluorinated products"],
+            "column_names": [
+                "category",
+                "CO2",
+                "CH4",
+                "N2O",
+                "HFCs",
+                "Other fluorinated products",
+            ],
+            "cat_codes_manual": {
+                "2F. Use of products as substitutes for \nsubstances that degrade the ozone layer": "2.F",
+                "2B4. Production of caprolactam, \nglyoxal and glyoxylic acid": "2.B.4",
+            },
+            "unit_for_entity": {
+                "CO2": "Gg",
+                "CH4": "Gg",
+                "N2O": "Gg",
+                "HFCs": "Gg CO2eq",
+            },
+        },
+        "88": {
+            "skip_rows_start": 2,
+            "entities": ["CO2", "CH4", "N2O"],
+            "column_names": ["category", "CO2", "CH4", "N2O"],
+            "cat_codes_manual": {
+                "3C6. Indirect emissions of N²O from manure \nmanagement": "3.C.6",
+                "3C. Aggregate Sources and Sources of Non-CO²\nEmissions in the soil": "3.C",
+            },
+            "unit_for_entity": {
+                "CO2": "Gg",
+                "CH4": "Gg",
+                "N2O": "Gg",
+            },
+        },
+        "89": {
+            "skip_rows_start": 2,
+            "entities": ["CO2", "CH4", "N2O"],
+            "column_names": ["category", "CO2", "CH4", "N2O"],
+            "cat_codes_manual": {
+                "3C6. Indirect emissions of N²O from manure \nmanagement": "3.C.6",
+                "3C. Aggregate Sources and Sources of Non-CO²\nEmissions in the soil": "3.C",
+            },
+            "unit_for_entity": {
+                "CO2": "Gg",
+                "CH4": "Gg",
+                "N2O": "Gg",
+            },
+        },
+    },
+}
+
+country_processing_step1 = {
+    # rounding error 0.038 for yr2019/entN2O/cat4: 0.011 + 0.015 != 0.027
+    "tolerance": 0.04,
+    "aggregate_cats": {
+        # First generate additional categories
+        "0": {"sources": ["1", "2", "3", "4", "5"]},
+        "2.A": {"sources": ["2.A.1", "2.A.2", "2.A.3", "2.A.4", "2.A.5"]},
+        "2.C": {
+            "sources": [
+                "2.C.1",
+                "2.C.2",
+                "2.C.3",
+                "2.C.4",
+                "2.C.5",
+                "2.C.6",
+                "2.C.7",
+            ]
+        },
+        "2.E": {"sources": ["2.E.1", "2.E.2", "2.E.3", "2.E.4", "2.E.5"]},
+        "3.C": {
+            "sources": [
+                "3.C.1",
+                "3.C.2",
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.6",
+                "3.C.7",
+                "3.C.8",
+            ]
+        },
+        "3.D": {"sources": ["3.D.2"]},
+        # 3.D would go into M.LULUCF as well but we don't have it
+        "M.LULUCF": {"sources": ["3.B"]},
+        # Only 3.C.7 in table, but values are all zero or empty
+        "M.3.C.AG": {
+            "sources": [
+                "3.C.1",
+                "3.C.2",
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.6",
+                "3.C.7",
+                "3.C.8",
+            ]
+        },
+        # 3.D.2 is all zeros
+        "M.3.D.AG": {"sources": ["3.D.2"]},
+        "M.AG.ELV": {
+            "sources": ["M.3.C.AG", "M.3.D.AG"],
+        },
+        "M.AG": {"sources": ["3.A", "M.AG.ELV"]},
+        "M.0.EL": {
+            "sources": ["1", "2", "M.AG", "4"],
+        },
+        "4.D": {"sources": ["4.D.1", "4.D.2"]},  # consistency check
+        "1": {"sources": ["1.A"]},  # consistency check
+        "1.A": {
+            "sources": ["1.A.1", "1.A.2", "1.A.3", "1.A.4", "1.A.5"]
+        },  # consistency check
+        "2": {
+            "sources": ["2.A", "2.B", "2.C", "2.D", "2.E", "2.F", "2.G", "2.H"]
+        },  # consistency check
+        "3": {"sources": ["M.AG", "M.LULUCF"]},  # consistency check
+        # "3.A": {"sources": ["3.A.1", "3.A.2"]}, # consistency check
+        "4": {"sources": ["4.A", "4.B", "4.C", "4.D", "4.E"]},  # consistency check
+    },
+}
+
+country_processing_step2 = {
+    "downscale": {
+        "sectors": {
+            "1_all": {
+                "basket": "1",
+                "basket_contents": ["1.A"],
+                "entities": ["CO2", "CH4", "N2O"],
+                "dim": f'category ({coords_terminologies["category"]})',
+            },
+            # Values for 1995/2000/2005/2010/2015/2019 are only available for CO2 and F-gases (table 6)
+            "2_CO2": {
+                "basket": "2",
+                "basket_contents": ["2.A", "2.B", "2.C", "2.D", "2.H"],
+                "entities": ["CO2"],
+                "dim": f'category ({coords_terminologies["category"]})',
+            },
+            # "2_KYOTO": {  # commented as KYOTOGHG only present for bunkers, so aggregate later
+            #     # error of 3.5% for KYOTOGHG (AR6GWP100)
+            #     "check_consistency": False,
+            #     "basket": "2",
+            #     "basket_contents": [
+            #         "2.A",
+            #         "2.B",
+            #         "2.C",
+            #         "2.D",
+            #         "2.E",
+            #         "2.F",
+            #         "2.G",
+            #         "2.H",
+            #     ],
+            #     "entities": [
+            #         "KYOTOGHG (SARGWP100)",
+            #         "KYOTOGHG (AR4GWP100)",
+            #         "KYOTOGHG (AR5GWP100)",
+            #         "KYOTOGHG (AR6GWP100)",
+            #     ],
+            #     "dim": f'category ({coords_terminologies["category"]})',
+            # },
+            # "2_FGASES": {  # f-gases have no sectoral detail and HFCs only 2019,
+            #     # no downscaling of original data possible
+            #     # error of 3.5% for KYOTOGHG (AR6GWP100)
+            #     "check_consistency": False,
+            #     "basket": "2",
+            #     "basket_contents": [
+            #         "2.B",
+            #         "2.C",
+            #         "2.E",
+            #         "2.F",
+            #         "2.G",
+            #     ],
+            #     "entities": [
+            #         "FGASES (SARGWP100)",
+            #         "FGASES (AR4GWP100)",
+            #         "FGASES (AR5GWP100)",
+            #         "FGASES (AR6GWP100)",
+            #         "HFCS (SARGWP100)",
+            #         "HFCS (AR4GWP100)",
+            #         "HFCS (AR5GWP100)",
+            #         "HFCS (AR6GWP100)",
+            #     ],
+            #     "dim": f'category ({coords_terminologies["category"]})',
+            # },
+            "3_CH4": {
+                "basket": "3",
+                "basket_contents": ["3.A", "3.B", "3.C", "3.D"],
+                "entities": ["CH4"],
+                "dim": f'category ({coords_terminologies["category"]})',
+            },
+            "3_CO2": {
+                "basket": "3",
+                "basket_contents": ["3.B", "3.C", "3.D"],
+                "entities": ["CO2"],
+                "dim": f'category ({coords_terminologies["category"]})',
+            },
+            "3_N2O": {
+                "basket": "3",
+                "basket_contents": ["3.A", "3.B", "3.C", "3.D"],
+                "entities": ["N2O"],
+                "dim": f'category ({coords_terminologies["category"]})',
+            },
+            # "3_KYOTO": {  # no original data here. aggregate basket later
+            #     "basket": "3",
+            #     "basket_contents": ["3.A", "3.B", "3.C", "3.D"],
+            #     "entities": [
+            #         "KYOTOGHG (SARGWP100)",
+            #         "KYOTOGHG (AR4GWP100)",
+            #         "KYOTOGHG (AR5GWP100)",
+            #         "KYOTOGHG (AR6GWP100)",
+            #     ],
+            #     "dim": f'category ({coords_terminologies["category"]})',
+            # },
+        },
+        "entities": {
+            "FGASES": {
+                "basket": "FGASES (SARGWP100)",
+                "basket_contents": ["HFCS (SARGWP100)"],
+            },
+        },
+    },
+    "basket_copy": {
+        "GWPs_to_add": ["AR4GWP100", "AR5GWP100", "AR6GWP100"],
+        "entities": ["HFCS"],
+        "source_GWP": gwp_to_use,
+    },
+}
+
+gas_baskets = {
+    "FGASES (SARGWP100)": ["HFCS (SARGWP100)"],
+    "FGASES (AR4GWP100)": ["HFCS (AR4GWP100)"],
+    "FGASES (AR5GWP100)": ["HFCS (AR5GWP100)"],
+    "FGASES (AR6GWP100)": ["HFCS (AR6GWP100)"],
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O", "FGASES (SARGWP100)"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR4GWP100)"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR5GWP100)"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR6GWP100)"],
+}

src/unfccc_ghg_data/unfccc_reader/Cabo_Verde/read_CPV_BUR1_from_pdf.py

@@ -0,0 +1,331 @@
+"""
+Read Cabo Verde's BUR1 from pdf
+"""
+
+import camelot
+import numpy as np
+import pandas as pd
+import primap2 as pm2
+
+from unfccc_ghg_data.helper import (
+    downloaded_data_path,
+    extracted_data_path,
+    process_data_for_country,
+)
+from unfccc_ghg_data.unfccc_reader.Cabo_Verde.config_cpv_bur1 import (
+    coords_cols,
+    coords_defaults,
+    coords_terminologies,
+    coords_value_mapping,
+    coords_value_mapping_main,
+    country_processing_step1,
+    country_processing_step2,
+    filter_remove,
+    gas_baskets,
+    inv_conf,
+    inv_conf_main,
+    inv_conf_per_sector,
+    meta_data,
+    trend_years,
+)
+
+if __name__ == "__main__":
+    # ###
+    # configuration
+    # ###
+
+    # for regex later
+    def repl(m):  # noqa: D103
+        return m.group("code")
+
+    input_folder = downloaded_data_path / "UNFCCC" / "Cabo_Verde" / "BUR1"
+    output_folder = extracted_data_path / "UNFCCC" / "Cabo_Verde"
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+
+    pdf_file = "BUR_EN_Digital.pdf"
+    output_filename = "CPV_BUR1_2023_"
+    category_column = f"category ({coords_terminologies['category']})"
+    compression = dict(zlib=True, complevel=9)
+
+    # ###
+    # 1. Read sector-specific main tables for 2019
+    # ###
+
+    df_main = None
+    for page in reversed(inv_conf_main["pages"].keys()):
+        print(f"Read table on page {page}")
+        tables_inventory_original = camelot.read_pdf(
+            str(input_folder / pdf_file),
+            pages=page,
+            flavor="lattice",
+            split_text=True,
+        )
+
+        df_page = tables_inventory_original[0].df
+
+        skip_rows_start = inv_conf_main["pages"][page]["skip_rows_start"]
+        if not skip_rows_start == 0:
+            df_page = df_page[skip_rows_start:]
+
+        df_page.columns = inv_conf_main["pages"][page]["column_names"]
+
+        # first the manual replacements
+        df_page["category"] = df_page["category"].replace(
+            inv_conf_main["pages"][page]["cat_codes_manual"]
+        )
+
+        # Remove dots between letters in category codes
+        df_page["category"] = df_page["category"].str.replace(".", "")
+        # Some categories have a dash between the letters
+        df_page["category"] = df_page["category"].str.replace("-", " ")
+
+        # then the regex replacements
+        df_page["category"] = df_page["category"].str.replace(
+            inv_conf["cat_code_regexp"], repl, regex=True
+        )
+
+        df_page = pd.melt(
+            df_page,
+            id_vars="category",
+            value_vars=inv_conf_main["pages"][page]["entities"],
+        )
+
+        df_page = df_page.rename({"value": "data", "variable": "entity"}, axis=1)
+
+        df_page["data"] = df_page["data"].str.replace(",", ".")
+
+        # df_page["unit"] = df_page["entity"]
+
+        # set unit based on entity
+        df_page["unit"] = df_page["entity"].replace(
+            inv_conf_main["pages"][page]["unit_for_entity"]
+        )
+
+        # stack the tables vertically
+        if df_main is None:
+            df_main = df_page
+        else:
+            df_main = pd.concat(
+                [
+                    df_main,
+                    df_page,
+                ],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    df_main["time"] = inv_conf["year"]
+
+    # remove wrong codes in data column
+    df_main["data"] = df_main["data"].str.replace("HFC", "")
+
+    # Sum up the values for duplicate categories
+    cat = inv_conf["merge_cats"]
+    df_temp = df_main.loc[df_main["category"] == cat]
+    df_temp["data"] = df_temp["data"].replace("", np.nan).apply(float)
+    df_temp = df_temp.groupby(
+        ["category", "entity", "unit", "time"], as_index=False
+    ).sum()
+    # change back to empty strings
+    df_temp = df_temp.replace(0, "")
+    # drop category from df
+    df_main = df_main.drop(df_main[df_main["category"] == cat].index)
+    # append the summed up sub-set
+    df_main = pd.concat(
+        [df_main, df_temp],
+        axis=0,
+        join="outer",
+    ).reset_index(drop=True)
+
+    df_main_if = pm2.pm2io.convert_long_dataframe_if(
+        df_main,
+        coords_cols=coords_cols,
+        # add_coords_cols=add_coords_cols,
+        coords_defaults=coords_defaults,
+        coords_terminologies=coords_terminologies,
+        coords_value_mapping=coords_value_mapping_main,
+        # coords_value_filling=coords_value_filling,
+        filter_remove=filter_remove,
+        # filter_keep=filter_keep,
+        meta_data=meta_data,
+        convert_str=True,
+        time_format="%Y",
+    )
+
+    ### convert to primap2 format ###
+    print("Converting to primap2 format.")
+    data_main_pm2 = pm2.pm2io.from_interchange_format(df_main_if)
+
+    # ###
+    # 2. Read trend tables 1995, 2000, 2005, 2010, 2015 and 2019
+    # ###
+    df_trend = None
+    for sector in reversed(inv_conf_per_sector.keys()):
+        tables_inventory_original = camelot.read_pdf(
+            str(input_folder / pdf_file),
+            pages=inv_conf_per_sector[sector]["page"],
+            flavor="lattice",
+            split_text=True,
+        )
+
+        df_page = tables_inventory_original[0].df
+
+        # cut rows at the top if needed
+        skip_rows_start = inv_conf_per_sector[sector]["skip_rows_start"]
+        if not skip_rows_start == 0:
+            df_page = df_page[skip_rows_start:]
+
+        # drop columns if needed
+        if "drop_cols" in inv_conf_per_sector[sector].keys():
+            # print(df_current.columns.to_numpy())
+            df_page = df_page.drop(columns=inv_conf_per_sector[sector]["drop_cols"])
+
+        df_page.columns = inv_conf_per_sector[sector]["header"]
+
+        # fill empty strings with NaN and the forward fill category names
+        df_page["category"] = df_page["category"].replace("", np.nan).ffill()
+
+        # remove /n from category names
+        df_page["category"] = df_page["category"].str.replace("\n", "")
+        # manual replacement of categories
+        df_page["category"] = df_page["category"].replace(
+            inv_conf_per_sector[sector]["cat_codes_manual"]
+        )
+
+        # remove all thousand separator commas
+        for year in trend_years:
+            df_page[year] = df_page[year].str.replace(",", ".")
+
+        # add unit
+        df_page["unit"] = inv_conf_per_sector[sector]["unit"]
+
+        # add entity if needed
+        if "entity" in inv_conf_per_sector[sector].keys():
+            df_page["entity"] = inv_conf_per_sector[sector]["entity"]
+
+        if "unit_conversion" in inv_conf_per_sector[sector].keys():
+            for year in trend_years:
+                index = inv_conf_per_sector[sector]["unit_conversion"]["index"]
+                conv_factor = inv_conf_per_sector[sector]["unit_conversion"][
+                    "conversion_factor"
+                ]
+                df_page.loc[index, year] = str(
+                    conv_factor * float(df_page.loc[index, year])
+                )
+
+        # stack the tables vertically
+        if df_trend is None:
+            df_trend = df_page
+        else:
+            df_trend = pd.concat(
+                [
+                    df_trend,
+                    df_page,
+                ],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    df_trend_if = pm2.pm2io.convert_wide_dataframe_if(
+        df_trend,
+        coords_cols=coords_cols,
+        # add_coords_cols=add_coords_cols,
+        coords_defaults=coords_defaults,
+        coords_terminologies=coords_terminologies,
+        coords_value_mapping=coords_value_mapping,
+        # coords_value_filling=coords_value_filling,
+        filter_remove=filter_remove,
+        # filter_keep=filter_keep,
+        meta_data=meta_data,
+    )
+
+    ### convert to primap2 format ###
+    print("Converting to primap2 format.")
+    data_trend_pm2 = pm2.pm2io.from_interchange_format(df_trend_if)
+
+    # ###
+    # Merge the main table for 2019 and the trend tables
+    # ###
+
+    print("Merging main table and trend tables")
+    print("Merging waste table.")
+    data_pm2 = data_main_pm2.pr.merge(data_trend_pm2)  # , tolerance=0.10)
+
+    # # ###
+    # # Save raw data to IF and native format.
+    # # ###
+
+    data_if = data_pm2.pr.to_interchange_format()
+
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + coords_terminologies["category"] + "_raw"),
+        data_if,
+    )
+
+    encoding = {var: compression for var in data_pm2.data_vars}
+    data_pm2.pr.to_netcdf(
+        output_folder
+        / (output_filename + coords_terminologies["category"] + "_raw.nc"),
+        encoding=encoding,
+    )
+
+    # # ###
+    # # Processing
+    # # ###
+
+    # create the gas baskets before aggregating the categories
+    # data_proc_pm2_gas_baskets = process_data_for_country(
+    #     data_country=data_pm2,
+    #     entities_to_ignore=[],
+    #     gas_baskets=gas_baskets,
+    #     filter_dims=None,
+    #     cat_terminology_out=None,
+    #     category_conversion=None,
+    #     sectors_out=None,
+    #     processing_info_country=None,
+    # )
+
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_pm2,
+        entities_to_ignore=[],
+        gas_baskets={},  # gas_baskets,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step1,
+    )
+
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_proc_pm2,
+        entities_to_ignore=[],
+        gas_baskets=gas_baskets,  # None,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step2,
+    )
+
+    # # ###
+    # # save processed data to IF and native format
+    # # ###
+
+    terminology_proc = coords_terminologies["category"]
+
+    data_proc_if = data_proc_pm2.pr.to_interchange_format()
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + terminology_proc), data_proc_if
+    )
+
+    encoding = {var: compression for var in data_proc_pm2.data_vars}
+    data_proc_pm2.pr.to_netcdf(
+        output_folder / (output_filename + terminology_proc + ".nc"), encoding=encoding
+    )
+
+    print("Saved processed data.")

src/unfccc_ghg_data/unfccc_reader/Saint_Kitts_and_Nevis/__init__.py

@@ -0,0 +1,30 @@
+"""Saint Kitts and Nevis' BURs, NIRs, NCs
+
+Scripts and configurations to read Argentina's submissions to the UNFCCC.
+Currently, the following submissions and datasets are available (all datasets
+including DI (red using the DI-reader) and legacy BUR/NIR (no code)):
+
+.. exec_code::
+    :hide_code:
+
+    from unfccc_ghg_data.helper.functions import (get_country_datasets,
+                                                  get_country_submissions)
+    country = 'KNA'
+    # print available submissions
+    print("="*15 + " Available submissions " + "="*15)
+    get_country_submissions(country, True)
+    print("")
+
+    #print available datasets
+    print("="*15 + " Available datasets " + "="*15)
+    get_country_datasets(country, True)
+
+You can also obtain this information running
+
+.. code-block:: bash
+
+    poetry run doit country_info country=KNA
+
+See below for a listing of scripts for BUR/NIR reading including links.
+
+"""

src/unfccc_ghg_data/unfccc_reader/Saint_Kitts_and_Nevis/config_kna_bur1.py

@@ -0,0 +1,644 @@
+"""
+Configuration file to read Saint Kitts and Nevis' BUR 1.
+"""
+
+gwp_to_use = "AR5GWP100"
+
+# primap2 format conversion
+coords_cols = {
+    "category": "category",
+    "entity": "entity",
+    "unit": "unit",
+}
+
+coords_defaults = {
+    "source": "KNA-GHG-Inventory",
+    "provenance": "measured",
+    "area": "KNA",
+    "scenario": "BUR1",
+}
+
+coords_terminologies = {
+    "area": "ISO3",
+    "category": "IPCC2006_PRIMAP",
+    "scenario": "PRIMAP",
+}
+
+coords_value_mapping = {
+    "unit": "PRIMAP1",
+    "category": "PRIMAP1",
+    "entity": {
+        "NMVOCs": "NMVOC",
+        "HFCS": f"HFCS ({gwp_to_use})",
+        "PFCS": f"PFCS ({gwp_to_use})",
+        "SF6": f"SF6 ({gwp_to_use})",
+        "Other halogenated gases with CO2 equivalent conversion factors (1)": f"UnspMixOfHFCs ({gwp_to_use})",
+    },
+}
+
+meta_data = {
+    "references": "https://unfccc.int/documents/633382",
+    "rights": "",  # unknown
+    "contact": "daniel-busch@climate-resource.de",
+    "title": "Saint Kitts and Nevis. Biennial update report (BUR). BUR1",
+    "comment": "Read fom pdf by Daniel Busch",
+    "institution": "UNFCCC",
+}
+
+filter_remove = {
+    "f_memo": {"category": "MEMO"},
+    "f1": {
+        "entity": "Other halogenated gases without CO2 equivalent conversion factors (2)"
+    },
+    "f2": {"entity": "3D2LULUCF"},
+}
+
+conf_general = {
+    "cat_code_regexp": r"^(?P<code>[a-zA-Z0-9\.]{1,11})[\s\.].*",
+}
+
+conf_trend = {
+    "fugitive": {
+        "rows_to_fix": {2: ["1.B.3 - Other emissions from"]},
+        "page_defs": {
+            "125": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 1,
+            },
+            "126": {
+                "read_params": dict(
+                    flavor="stream",
+                    table_areas=["72,681,564,638"],
+                    columns=["203,238,272,305,340,370,402,439,469,504,536"],
+                ),
+                "skip_rows_start": 1,
+            },
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+    },
+    "other_sectors": {
+        "page_defs": {
+            "123": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 2,
+            },
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+    },
+    "transport_sub": {
+        "page_defs": {
+            "121": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 2,
+            },
+            "122": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 0,
+            },
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+    },
+    "transport": {
+        "page_defs": {
+            "119": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 2,
+            }
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+    },
+    "manufacturing_and_construction": {
+        "page_defs": {
+            "118": {
+                "read_params": dict(flavor="lattice"),
+                "skip_rows_start": 2,
+            }
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+    },
+    "energy_industries": {
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "replace_data_entries": {"NO,NE": "NO"},
+        "cat_codes_manual": {
+            "a. Public electricity and heat production": "1.A.1.a",
+            "b. Petroleum refining": "1.A.1.b",
+            "c. Manufacture of solid fuels": "1.A.1.c",
+        },
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [],
+        "rows_to_fix": {3: ["a. Public electricity and heat"]},
+        "page_defs": {
+            "116": {
+                "read_params": dict(
+                    flavor="stream",
+                    table_areas=["72,426,543,333"],
+                    columns=["199,229,261,293,324,356,386,416,448,480,511"],
+                ),
+                "skip_rows_start": 2,
+            },
+        },
+    },
+    "overview": {
+        # Inconsistencies for table page 11 and page 125 for categories 1.B, 1.B.1
+        "rows_to_drop": ["1B", "1B1"],
+        "fix_single_value": {
+            "cat": "MBIO",
+            "year": "2018",
+            "new_value": "0.17",
+        },
+        "entity": f"KYOTOGHG ({gwp_to_use})",
+        "unit": "GgCO2eq",
+        "replace_data_entries": {"NO,NE": "NO"},
+        "cat_codes_manual": {
+            "Total CO2 Eq. Emissions without  LULUCF": "M.0.EL",
+            "Total CO2 Eq. Emissions with  LULUCF": "0",
+            # "1. Energy": "1. Energy",
+            "A. Fuel Combustion": "1.A",
+            "1.  Energy Industries": "1.A.1",
+            "2.  Man. Ind. & Constr.": "1.A.2",
+            "3.  Transport": "1.A.3",
+            "4.  Other Sectors": "1.A.4",
+            "5.  Other": "1.A.5",
+            "B. Fugitive Emissions from Fuels": "1.B",
+            "1.  Solid Fuels": "1.B.1",
+            "2.  Oil and Natural Gas and other…": "M.1.B.23",
+            # "2.  Industrial Processes": "2.  Industrial Processes",
+            "A.  Mineral Industry": "2.A",
+            "B.  Chemical Industry": "2.B",
+            "C.  Metal Industry": "2.C",
+            "D.  Non-energy products": "2.D",
+            "E.  Electronics industry": "2.E",
+            "F.  Product uses as ODS substitutes": "2.F",
+            "G.  Other product manufacture and": "2.G",
+            "use  H.  Other": "2.H",
+            "3.  Agriculture": "M.AG",
+            "A.  Enteric Fermentation": "3.A.1",
+            "B.  Manure Management": "3.A.2",
+            "C.  Rice Cultivation": "3.C.7",
+            "D.  Agricultural Soils": "3.C.4",
+            "E.  Prescribed Burning of Savannahs": "3.C.1.c",
+            "F.  Field Burning of Agricultural": "3.C.1.b",
+            "Residues  G.  Liming": "3.C.2",
+            "H.  Urea applications": "3.C.3",
+            "I.  Other carbon-containing": "M.3.D.2.AG",
+            "fertilisers  4. Land Use, Land-Use Change and  Forestry": "M.LULUCF",
+            "A. Forest Land": "3.B.1",
+            "B. Cropland": "3.B.2",
+            "C. Grassland": "3.B.3",
+            "D. Wetlands": "3.B.4",
+            "E. Settlements": "3.B.5",
+            "F. Other Land": "3.B.6",
+            "G. Harvested wood products": "3.D.1",
+            "H. Other": "3.D.2.LULUCF",
+            "5. Waste": "4",
+            "A.  Solid Waste Disposal": "4.A",
+            "B.  Biological treatment of solid": "4.B",
+            "waste  C. Incineration and open burning of": "4.C",
+            "D. Waste water treatment and": "4.D",
+            "discharge  E.  Other": "4.E",
+            "6.  Other": "5",
+            "CO2 Emissions from Biomass": "M.BIO",
+        },
+        "drop_cols": [
+            "change to BY",
+            "change to PY",
+        ],
+        "header": ["orig_category"],
+        "years": [
+            "2008",
+            "2009",
+            "2010",
+            "2011",
+            "2012",
+            "2013",
+            "2014",
+            "2015",
+            "2016",
+            "2017",
+            "2018",
+        ],
+        "extra_columns": [
+            "change to BY",
+            "change to PY",
+        ],
+        "split_values": {
+            "cat": "3B2",
+            "keep_value_no": 1,
+        },
+        "page_defs": {
+            "111": {"read_params": dict(flavor="lattice"), "skip_rows_start": 1},
+            "112": {"read_params": dict(flavor="lattice"), "skip_rows_start": 1},
+            "113": {"read_params": dict(flavor="lattice"), "skip_rows_start": 1},
+        },
+    },
+}
+
+conf = {
+    "energy": {
+        "entities": ["CO2", "CH4", "N2O", "NOX", "CO", "NMVOCs", "SO2"],
+        "header": ["orig_category"],
+        "cat_codes_manual": {
+            "Information Items": "MEMO",
+            "CO2 from Biomass Combustion for Energy Production": "MBIO",
+        },
+        "page_defs": {
+            "149": {"skip_rows_start": 2},
+            "150": {"skip_rows_start": 2},
+            "151": {"skip_rows_start": 2},
+            "152": {"skip_rows_start": 2},
+        },
+        "replace_data_entries": {
+            "NO,NE": "NO",
+            "NE,NO": "NO",
+            "NO,IE": "NO",
+        },
+        "unit_mapping": {
+            "CO2": "Gg",
+            "CH4": "Gg",
+            "N2O": "Gg",
+            "NOX": "Gg",
+            "CO": "Gg",
+            "NMVOCs": "Gg",
+            "SO2": "Gg",
+        },
+    },
+    "ipuu": {
+        "entities": [
+            "CO2",
+            "CH4",
+            "N2O",
+            "HFCS",
+            "PFCS",
+            "SF6",
+            "Other halogenated gases with CO2 equivalent conversion factors (1)",
+            "Other halogenated gases without CO2 equivalent conversion factors (2)",
+            "NOX",
+            "CO",
+            "NMVOC",
+            "SO2",
+        ],
+        "header": ["orig_category"],
+        "cat_codes_manual": {
+            "Information Items": "MEMO",
+            "CO2 from Biomass Combustion for Energy Production": "MBIO",
+        },
+        "page_defs": {
+            "153": {"skip_rows_start": 2},
+            "154": {"skip_rows_start": 2},
+            "155": {"skip_rows_start": 2},
+        },
+        "replace_data_entries": {
+            "NO,NE": "NO",
+            "NE,NO": "NO",
+            "NO,IE": "NO",
+        },
+        "unit_mapping": {
+            "CO2": "Gg",
+            "CH4": "Gg",
+            "N2O": "Gg",
+            "HFCS": "GgCO2eq",
+            "PFCS": "GgCO2eq",
+            "SF6": "GgCO2eq",
+            "Other halogenated gases with CO2 equivalent conversion factors (1)": "GgCO2eq",
+            "Other halogenated gases without CO2 equivalent conversion factors (2)": "Gg",
+            "NOX": "Gg",
+            "CO": "Gg",
+            "NMVOC": "Gg",
+            "SO2": "Gg",
+        },
+    },
+    "AFOLU": {
+        "entities": [
+            "CO2",
+            "CH4",
+            "N2O",
+            "NOX",
+            "CO",
+            "NMVOC",
+        ],
+        "header": ["orig_category"],
+        "cat_codes_manual": {
+            "Information Items": "MEMO",
+            "CO2 from Biomass Combustion for Energy Production": "MBIO",
+        },
+        "page_defs": {
+            "156": {"skip_rows_start": 3},
+            "157": {"skip_rows_start": 3},
+            "158": {"skip_rows_start": 3},
+        },
+        "replace_data_entries": {
+            "NO,NA": "NO",
+            "NO,NE": "NO",
+            "NE,NO": "NO",
+            "NO,IE": "NO",
+        },
+        "unit_mapping": {
+            "CO2": "Gg",
+            "CH4": "Gg",
+            "N2O": "Gg",
+            "NOX": "Gg",
+            "CO": "Gg",
+            "NMVOC": "Gg",
+        },
+    },
+    "waste": {
+        "entities": [
+            "CO2",
+            "CH4",
+            "N2O",
+            "NOX",
+            "CO",
+            "NMVOC",
+            "SO2",
+        ],
+        "header": ["orig_category"],
+        "cat_codes_manual": {
+            "Information Items": "MEMO",
+            "CO2 from Biomass Combustion for Energy Production": "MBIO",
+        },
+        "page_defs": {
+            "159": {"skip_rows_start": 2},
+        },
+        "replace_data_entries": {
+            "NO,NA": "NO",
+            "NO,NE": "NO",
+            "NE,NO": "NO",
+            "NO,IE": "NO",
+        },
+        "unit_mapping": {
+            "CO2": "Gg",
+            "CH4": "Gg",
+            "N2O": "Gg",
+            "NOX": "Gg",
+            "CO": "Gg",
+            "NMVOC": "Gg",
+            "SO2": "Gg",
+        },
+    },
+}
+
+fix_values_main = [
+    # numbers don't add up for 3.A
+    ("3A2", "CH4", "0.03"),
+    ("3A2", "N2O", "0"),
+    # numbers don't add up for 1.B
+    ("1B2a", "CO2", "0.002288"),  # value from 1.B nowhere in sub-categories
+    ("1B2aiii", "CO2", "0.002288"),  # value from 1.B.2.a nowhere in sub-categories
+    ("1B2aiii3", "CO2", "0.002288"),  # value from 1.B.2.a.iii nowhere in sub-categories
+]
+
+fix_values_trend = [
+    # Most of the values for (KYOTOGHG (AR5GWP100)) don't match
+    # with the values from the main table.
+    # Replacing with values from main table
+    # energy
+    ("1A3bi", "2018", "64.74"),  # (category, year, new_value)
+    ("1A3bi1", "2018", "64.7"),
+    ("1A3bii", "2018", "12.36"),
+    ("1A3bii1", "2018", "11.07"),
+    ("1A3bii2", "2018", "1.28"),
+    ("1A3biii", "2018", "23.66"),
+    ("1A3biv", "2018", "0.16"),
+    ("1A3c", "2018", "0.17"),
+    ("1B", "2018", "0.002288"),
+    ("1B2", "2018", "0.002288"),
+    ("1B2a", "2018", "0.002288"),
+    ("1B2aiii", "2018", "0.002288"),
+    ("1B2aiii3", "2018", "0.002288"),
+    # agriculture
+    ("3A1", "2018", "5.04"),
+    ("3A2", "2018", "0.84"),
+    ("3C4", "2018", "2.65"),
+    ("MAG", "2018", "8.54"),
+    # lulucf
+    # There are missing numbers in "Forest Land" - 3.B.1 on page 112
+    # I found them as invisible numbers in the row below
+    # but deleted them because I didn't know where they belong.
+    # Leaving it as it is now, but numbers could be added upstream TODO
+    ("3B1", "2008", "-130.02"),
+    ("3B1", "2009", "-130.02"),
+    ("3B1", "2010", "-130.02"),
+    ("3B1", "2011", "-151.6"),
+    ("3B1", "2012", "-151.6"),
+    ("3B1", "2013", "-151.6"),
+    ("3B1", "2014", "-140.34"),
+    ("3B1", "2015", "-140.34"),
+    ("3B1", "2016", "-140.34"),
+    ("3B1", "2017", "-140.34"),
+    ("3B1", "2018", "-140.34"),
+    # waste
+    ("4D", "2018", "12.32"),
+    ("4C", "2018", "0.03"),
+    ("4A", "2018", "45.92"),
+    ("4", "2018", "58.27"),
+]
+
+gas_baskets = {
+    "FGASES (SARGWP100)": ["HFCS (SARGWP100)", "PFCS (SARGWP100)", "SF6", "NF3"],
+    "FGASES (AR4GWP100)": ["HFCS (AR4GWP100)", "PFCS (AR4GWP100)", "SF6", "NF3"],
+    "FGASES (AR5GWP100)": ["HFCS (AR5GWP100)", "PFCS (AR5GWP100)", "SF6", "NF3"],
+    "FGASES (AR6GWP100)": ["HFCS (AR6GWP100)", "PFCS (AR6GWP100)", "SF6", "NF3"],
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O", "FGASES (SARGWP100)"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR4GWP100)"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR5GWP100)"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR6GWP100)"],
+}
+
+country_processing_step1 = {
+    "tolerance": 0.01,
+    "aggregate_cats": {
+        "M.3.D.AG": {"sources": ["M.3.D.2.AG"]},
+        "3.C.1.AG": {"sources": ["3.C.1.b", "3.C.1.c"]},
+        "M.3.C.AG": {
+            "sources": [
+                "3.C.1.AG",
+                "3.C.2",
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.6",
+                "3.C.7",
+                "3.C.8",
+            ],
+        },
+        "M.AG.ELV": {
+            "sources": ["M.3.C.AG", "M.3.D.AG"],
+        },
+        "3.A": {"sources": ["3.A.1", "3.A.2"]},
+        "3.B": {"sources": ["3.B.1", "3.B.2", "3.B.3", "3.B.4", "3.B.5", "3.B.6"]},
+        "3.C": {
+            "sources": [
+                "3.C.1",
+                "3.C.2",
+                "3.C.3",
+                "3.C.4",
+                "3.C.5",
+                "3.C.6",
+                "3.C.7",
+                "3.C.8",
+            ]
+        },
+        "3.D": {"sources": ["3.D.1", "3.D.2"]},
+        "M.AG": {"sources": ["3.A", "M.AG.ELV"]},
+        "3.C.1.LU": {"sources": ["3.C.1.a", "3.C.1.d"]},
+        "M.3.D.LU": {"sources": ["3.D.1"]},
+        "M.LULUCF": {"sources": ["3.B", "3.C.1.LU", "M.3.D.LU"]},
+        "M.0.EL": {
+            "sources": ["1", "2", "M.AG", "4"],
+        },
+        "3": {"sources": ["M.AG", "M.LULUCF"]},  # consistency check
+        "0": {"sources": ["1", "2", "3", "4", "5"]},  # consistency check
+    },
+    "basket_copy": {
+        "GWPs_to_add": ["AR4GWP100", "SARGWP100", "AR6GWP100"],
+        "entities": ["HFCS", "PFCS", "UnspMixOfHFCs"],
+        "source_GWP": gwp_to_use,
+    },
+}
+
+country_processing_step2 = {
+    "downscale": {
+        # "sectors": {
+        #     "M.1.B.23": {
+        #         "basket": "M.1.B.23",
+        #         "basket_contents": ["1.B.2", "1.B.3"],
+        #         "entities": ["KYOTOGHG (AR5GWP100)"],
+        #         "dim": f'category ({coords_terminologies["category"]})',
+        #     },
+        # },
+        "entities": {
+            "KYOTO": {
+                "basket": "KYOTOGHG (AR5GWP100)",
+                "basket_contents": [
+                    "CH4",
+                    "CO2",
+                    "N2O",
+                    "HFCS (AR5GWP100)",
+                    "PFCS (AR5GWP100)",
+                    "SF6",
+                ],
+                "sel": {
+                    f'category ({coords_terminologies["category"]})': [
+                        "1",
+                        "1.A",
+                        "1.B",
+                        "1.B.2",
+                        # "1.B.3" # all zero -> doesn't work
+                        # "1.C",  # we don't have trend values for 1.C
+                        # Downscaling currently doesn't work for all zero basket content, see
+                        # https://github.com/pik-primap/primap2/issues/254#issue-2491434285
+                        # "2",  # all zero -> doesn't work
+                        # "2.A",  # all zero -> doesn't work
+                        # "2.B",  # all zero -> doesn't work
+                        # "2.C",  # all zero -> doesn't work
+                        # "2.D",  # all zero -> doesn't work
+                        # "2.E",  # all zero -> doesn't work
+                        # "2.F",  # all zero -> doesn't work
+                        # "2.G",  # all zero -> doesn't work
+                        # "2.H",  # all zero -> doesn't work
+                        "3",
+                        "3.A",
+                        "3.B",
+                        "3.C",
+                        "3.D",
+                        "4",
+                    ]
+                },
+            },
+        },
+    },
+}

src/unfccc_ghg_data/unfccc_reader/Saint_Kitts_and_Nevis/read_KNA_BUR1_from_pdf.py

@@ -0,0 +1,395 @@
+"""
+Read Saint Kitts and Nevis' BUR1 from pdf
+"""
+import camelot
+import pandas as pd
+import primap2 as pm2
+
+from unfccc_ghg_data.helper import (
+    downloaded_data_path,
+    extracted_data_path,
+    fix_rows,
+    process_data_for_country,
+)
+from unfccc_ghg_data.unfccc_reader.Saint_Kitts_and_Nevis.config_kna_bur1 import (
+    conf,
+    conf_general,
+    conf_trend,
+    coords_cols,
+    coords_defaults,
+    coords_terminologies,
+    coords_value_mapping,
+    country_processing_step1,
+    country_processing_step2,
+    filter_remove,
+    fix_values_main,
+    fix_values_trend,
+    gas_baskets,
+    meta_data,
+)
+
+if __name__ == "__main__":
+    # ###
+    # configuration
+    # ###
+
+    input_folder = downloaded_data_path / "UNFCCC" / "Saint_Kitts_and_Nevis" / "BUR1"
+    output_folder = extracted_data_path / "UNFCCC" / "Saint_Kitts_and_Nevis"
+    if not output_folder.exists():
+        output_folder.mkdir()
+
+    pdf_file = "First_BUR_St.Kitts_Nevis.pdf"
+    output_filename = "KNA_BUR1_2023_"
+    compression = dict(zlib=True, complevel=9)
+
+    def repl(m):  # noqa: D103
+        return m.group("code")
+
+    # ###
+    # 1. Read trend tables
+    # ###
+
+    df_trend = None
+    for table in conf_trend.keys():
+        print("-" * 45)
+        print(f"Reading {table} trend table.")
+        df_table = None
+        for page in conf_trend[table]["page_defs"].keys():
+            print(f"Page {page}")
+            tables_inventory_original = camelot.read_pdf(
+                str(input_folder / pdf_file),
+                pages=page,
+                split_text=True,
+                **conf_trend[table]["page_defs"][page]["read_params"],
+            )
+
+            df_page = tables_inventory_original[0].df
+
+            skip_rows_start = conf_trend[table]["page_defs"][page]["skip_rows_start"]
+            if not skip_rows_start == 0:
+                df_page = df_page[skip_rows_start:]
+
+            if df_table is None:
+                # Reset index to avoid pandas' SettingWithCopyWarning
+                df_table = df_page.reset_index(drop=True)
+            else:
+                df_table = pd.concat(
+                    [
+                        df_table,
+                        df_page,
+                    ],
+                    axis=0,
+                    join="outer",
+                ).reset_index(drop=True)
+
+        # fix content that spreads across multiple rows
+        if "rows_to_fix" in conf_trend[table]:
+            for n_rows in conf_trend[table]["rows_to_fix"].keys():
+                print(f"Merge content for {n_rows=}")
+                df_table = fix_rows(
+                    df_table,
+                    rows_to_fix=conf_trend[table]["rows_to_fix"][n_rows],
+                    col_to_use=0,
+                    n_rows=n_rows,
+                )
+
+        df_table.columns = (
+            conf_trend[table]["header"]
+            + conf_trend[table]["years"]
+            + conf_trend[table]["extra_columns"]
+        )
+
+        # drop columns if needed
+        if "drop_cols" in conf_trend[table].keys():
+            df_table = df_table.drop(columns=conf_trend[table]["drop_cols"])
+
+        # category codes from category names
+        df_table["category"] = df_table["orig_category"]
+        # Remove line break characters
+        df_table["category"] = df_table["category"].str.replace("\n", " ")
+        # first the manual replacements
+        if "cat_codes_manual" in conf_trend[table].keys():
+            df_table["category"] = df_table["category"].replace(
+                conf_trend[table]["cat_codes_manual"]
+            )
+        # remove dots from category codes
+        df_table["category"] = df_table["category"].str.replace(".", "")
+        # then the regex replacements
+        df_table["category"] = df_table["category"].str.replace(
+            conf_general["cat_code_regexp"], repl, regex=True
+        )
+        df_table = df_table.drop(columns="orig_category")
+
+        # drop rows if needed
+        if "rows_to_drop" in conf_trend[table].keys():
+            for row in conf_trend[table]["rows_to_drop"]:
+                row_to_delete = df_table.index[df_table["category"] == row][0]
+                df_table = df_table.drop(index=row_to_delete)
+
+        # bring values in right format
+        for year in conf_trend[table]["years"]:
+            if "replace_data_entries" in conf_trend[table].keys():
+                df_table[year] = df_table[year].replace(
+                    conf_trend[table]["replace_data_entries"]
+                )
+            df_table[year] = df_table[year].str.replace("\n", "")
+            df_table[year] = df_table[year].str.replace(",", ".")
+            # There are "invisible" numbers in trend table on page 112, "A. Forest Land"
+            # I'm removing them here, but they actually belong to the above,
+            # which I didn't know when I wrote this code
+            # TODO: Invisible values can be added to row above directly
+            if "split_values" in conf_trend[table].keys():
+                cat = conf_trend[table]["split_values"]["cat"]
+                keep_value_no = conf_trend[table]["split_values"]["keep_value_no"]
+                new_value = (
+                    df_table.loc[df_table["category"] == cat, year]
+                    .item()
+                    .split(" ")[keep_value_no]
+                )
+                df_table.loc[df_table["category"] == cat, year] = new_value
+
+        if "fix_single_value" in conf_trend[table].keys():
+            cat = conf_trend[table]["fix_single_value"]["cat"]
+            year = conf_trend[table]["fix_single_value"]["year"]
+            new_value = conf_trend[table]["fix_single_value"]["new_value"]
+            df_table.loc[df_table["category"] == cat, year] = new_value
+
+        df_table["unit"] = conf_trend[table]["unit"]
+        df_table["entity"] = conf_trend[table]["entity"]
+
+        # stack the tables vertically
+        if df_trend is None:
+            df_trend = df_table.reset_index(drop=True)
+        else:
+            df_trend = pd.concat(
+                [
+                    df_trend,
+                    df_table,
+                ],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    # some categories present in trend table on page 112 and the following detailed
+    # tables for the sub-categories
+    df_trend = df_trend.drop_duplicates()
+
+    for cat, year, new_value in fix_values_trend:
+        # make sure there is exactly one value that matches the filter
+        # TODO ruff wants to remove the assert statements here
+        assert len(df_trend.loc[df_trend["category"] == cat, year]) == 1  # noqa: S101
+        df_trend.loc[df_trend["category"] == cat, year] = new_value
+
+    df_trend_if = pm2.pm2io.convert_wide_dataframe_if(
+        df_trend,
+        coords_cols=coords_cols,
+        coords_defaults=coords_defaults,
+        coords_terminologies=coords_terminologies,
+        coords_value_mapping=coords_value_mapping,
+        filter_remove=filter_remove,
+        meta_data=meta_data,
+    )
+    #
+    ### convert to primap2 format ###
+    print("Converting to primap2 format.")
+    data_trend_pm2 = pm2.pm2io.from_interchange_format(df_trend_if)
+
+    # ###
+    # 2. Read in main tables
+    # ###
+
+    df_main = None
+    for sector in conf.keys():
+        print("-" * 45)
+        print(f"Reading table for {sector}.")
+
+        df_sector = None
+        for page in conf[sector]["page_defs"].keys():
+            print(f"Page {page}")
+            tables_inventory_original = camelot.read_pdf(
+                str(input_folder / pdf_file),
+                pages=page,
+                flavor="lattice",
+            )
+
+            df_page = tables_inventory_original[0].df
+
+            skip_rows_start = conf[sector]["page_defs"][page]["skip_rows_start"]
+            if not skip_rows_start == 0:
+                df_page = df_page[skip_rows_start:]
+
+            if df_sector is None:
+                # Reset index to avoid pandas' SettingWithCopyWarning
+                df_sector = df_page.reset_index(drop=True)
+            else:
+                df_sector = pd.concat(
+                    [
+                        df_sector,
+                        df_page,
+                    ],
+                    axis=0,
+                    join="outer",
+                ).reset_index(drop=True)
+
+        df_sector.columns = conf[sector]["header"] + conf[sector]["entities"]
+
+        df_sector["category"] = df_sector["orig_category"]
+
+        # Remove line break characters
+        df_sector["category"] = df_sector["category"].str.replace("\n", " ")
+
+        # first the manual replacements
+        df_sector["category"] = df_sector["category"].replace(
+            conf[sector]["cat_codes_manual"]
+        )
+
+        # remove dots from category codes
+        df_sector["category"] = df_sector["category"].str.replace(".", "")
+
+        # then the regex replacements
+        df_sector["category"] = df_sector["category"].str.replace(
+            conf_general["cat_code_regexp"], repl, regex=True
+        )
+
+        df_sector = df_sector.drop(columns="orig_category")
+
+        # bring in long format, so it can be concatenated with other tables
+        df_sector = pd.melt(
+            df_sector,
+            id_vars="category",
+            value_vars=conf[sector]["entities"],
+        )
+
+        # pd.melt always outputs value and variable as column names
+        df_sector = df_sector.rename({"value": "data", "variable": "entity"}, axis=1)
+
+        # clean data column
+        df_sector["data"] = df_sector["data"].replace(
+            conf[sector]["replace_data_entries"]
+        )
+        df_sector["data"] = df_sector["data"].str.replace(",", ".")
+        df_sector["data"] = df_sector["data"].str.replace("\n", "")
+
+        df_sector["unit"] = df_sector["entity"].replace(conf[sector]["unit_mapping"])
+
+        if df_main is None:
+            df_main = df_sector
+        else:
+            df_main = pd.concat(
+                [
+                    df_sector,
+                    df_main,
+                ],
+                axis=0,
+                join="outer",
+            ).reset_index(drop=True)
+
+    # year is the same for all sector tables
+    df_main["time"] = "2018"
+
+    # fix values
+    for cat, ent, new_value in fix_values_main:
+        # Make sure value to replace is found in data frame
+        # TODO ruff wants to remove the assert statements here
+        assert (  # noqa: S101
+            len(
+                df_main.loc[
+                    (df_main["category"] == cat) & (df_main["entity"] == ent), "data"
+                ]
+            )
+            == 1
+        )
+        df_main.loc[
+            (df_main["category"] == cat) & (df_main["entity"] == ent), "data"
+        ] = new_value
+
+    ### convert to interchange format ###
+    print("Converting to interchange format.")
+    df_main_IF = pm2.pm2io.convert_long_dataframe_if(
+        df_main,
+        coords_cols=coords_cols,
+        coords_defaults=coords_defaults,
+        coords_terminologies=coords_terminologies,
+        coords_value_mapping=coords_value_mapping,
+        filter_remove=filter_remove,
+        meta_data=meta_data,
+        convert_str=True,
+        time_format="%Y",
+    )
+
+    ### convert to primap2 format ###
+    print("Converting to primap2 format.")
+    data_main_pm2 = pm2.pm2io.from_interchange_format(df_main_IF)
+
+    # # ###
+    # # Merge tables.
+    # # ###
+
+    print("Merging main and trend table.")
+    data_pm2 = data_main_pm2.pr.merge(data_trend_pm2, tolerance=1)
+
+    # # ###
+    # # Save raw data to IF and native format.
+    # # ###
+
+    data_if = data_pm2.pr.to_interchange_format()
+
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + coords_terminologies["category"] + "_raw"),
+        data_if,
+    )
+
+    encoding = {var: compression for var in data_pm2.data_vars}
+    data_pm2.pr.to_netcdf(
+        output_folder
+        / (output_filename + coords_terminologies["category"] + "_raw.nc"),
+        encoding=encoding,
+    )
+
+    # # ###
+    # # Processing
+    # # ###
+
+    # create the gas baskets before aggregating the categories
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_pm2,
+        entities_to_ignore=[],
+        gas_baskets=gas_baskets,
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step1,
+    )
+
+    data_proc_pm2 = process_data_for_country(
+        data_country=data_proc_pm2,
+        entities_to_ignore=[],
+        gas_baskets={},
+        filter_dims=None,
+        cat_terminology_out=None,
+        category_conversion=None,
+        sectors_out=None,
+        processing_info_country=country_processing_step2,
+    )
+
+    # # ###
+    # # save processed data to IF and native format
+    # # ###
+
+    terminology_proc = coords_terminologies["category"]
+
+    data_proc_if = data_proc_pm2.pr.to_interchange_format()
+
+    if not output_folder.exists():
+        output_folder.mkdir()
+    pm2.pm2io.write_interchange_format(
+        output_folder / (output_filename + terminology_proc), data_proc_if
+    )
+
+    encoding = {var: compression for var in data_proc_pm2.data_vars}
+    data_proc_pm2.pr.to_netcdf(
+        output_folder / (output_filename + terminology_proc + ".nc"), encoding=encoding
+    )
+
+    print("Saved processed data.")

src/unfccc_ghg_data/unfccc_reader/Uzbekistan/__init__.py

@@ -0,0 +1,30 @@
+"""Read Uzbekistan's national inventories
+
+Scripts and configurations to read United Kingdoms's submissions to the UNFCCC.
+Currently, the following submissions and datasets are available (all datasets
+including DI (red using the DI-reader) and legacy BUR/NIR (no code)):
+
+.. exec_code::
+    :hide_code:
+
+    from unfccc_ghg_data.helper.functions import (get_country_datasets,
+                                                  get_country_submissions)
+    country = 'UZB'
+    # print available submissions
+    print("="*15 + " Available submissions " + "="*15)
+    get_country_submissions(country, True)
+    print("")
+
+    #print available datasets
+    print("="*15 + " Available datasets " + "="*15)
+    get_country_datasets(country, True)
+
+You can also obtain this information running
+
+.. code-block:: bash
+
+    poetry run doit country_info country=UZB
+
+See below for a listing of scripts for BUR/NIR reading including links.
+
+"""

src/unfccc_ghg_data/unfccc_reader/Uzbekistan/config_uzb_btr1.py

@@ -0,0 +1,2979 @@
+"""Config for UZB BTR1 (NIR, no xls files)
+
+General configuration for reading the NIR published as Uzbekistans BTR1.
+
+"""
+
+gwp_to_use = "AR4GWP100"
+cat_code_regexp = r"(?P<code>^[a-zA-Z0-9\.]{1,13})\s.*"
+
+## pdf reading config
+
+# there several gas mappings which look useless, but they are necessary
+# as the original values use characters that look the same but differ from
+# the normal characters and thus the units are not recognized without mapping
+table_def_trends = {
+    "M0EL_gases": {
+        "header": "entity",
+        "tables": {
+            "21": 0,
+        },
+        "split_kw": "Year",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.0.EL",
+            "unit": "Mt CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                # "HFC": f"HFCS ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%. 2021", "∆(2010-2021)"],
+            },
+            "fN2O": {  # indirect N2O from manure missing
+                "entity": ["N2O", "Total"],
+            },
+            "fHFC": {  # inconsistent with individual HFCs
+                "entity": ["HFC"],
+            },
+        },
+        "replace_str_data": {",": "."},
+        "remove_vals": {
+            "r1": {  # value inconsistent sector tables
+                "entities": ["CO2", "CH4"],
+                "filter": {"category": ["M.0.EL"], "time": ["2013"]},
+            }
+        },
+    },
+    "GHG_main_sectors": {
+        "header": "category",
+        "tables": {
+            "23": 0,
+        },
+        "split_kw": "Year",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Mt CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Energy": "1",
+                "IPPU": "2",
+                # "Agriculture": "M.AG",
+                "Waste": "4",
+                "Total": "M.0.EL",
+                "FOLU": "M.LULUCF",
+                "Total (excluding FOLU)": "0",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%, 2021", "∆(2010-2021)"],
+            },
+            "fAG": {  # indirect N2O from manure missing
+                "category": ["Agriculture"],
+            },
+        },
+        "replace_str_data": {" ": ""},
+        "remove_vals": {
+            "rounding_error": {  # rounding error
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["M.LULUCF"], "time": ["2002"]},
+            },
+            "r1": {  # value inconsistent sector tables
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["M.0.EL", "1"], "time": ["2013"]},
+            },
+        },
+    },
+    "0_prec": {
+        "header": "entity",
+        "tables": {
+            "24": 0,
+        },
+        "split_kw": "Year",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "0",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "СО": "CO",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+    },
+    "energy_gases": {
+        "header": "entity",
+        "tables": {
+            "29": 0,
+        },
+        "split_kw": "Year",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1",
+            "unit": "Mt CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆(1990−2021)"],
+            }
+        },
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": ["CH4", f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["1"], "time": ["2013"]},
+            }
+        },
+    },
+    "energy_prec": {
+        "tables": {
+            "29": 1,
+        },
+        "split_kw": "Gas",
+        "header": "entity",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "NOх": "NOx",
+                "СО": "CO",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            },
+            "f2": {
+                "entity": ["NMVOCs"],  # inconsistent with inventory and subsectors
+            },
+        },
+    },
+    "GHG_energy_sectors": {
+        "header": "category",
+        "tables": {
+            "30": 1,
+        },
+        "split_kw": "Category",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Mt CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Fuel Combustion": "1.A",
+                "Fugitive Emissions": "1.B",
+                "Fuel combustion": "1.A",
+                "Fugitive emissions": "1.B",
+                "Total": "1",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2017)", "%. 2021"],
+            }
+        },
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["1", "1.B"], "time": ["2013"]},
+            },
+        },
+    },
+    "bunkers_gases": {
+        "header": "entity",
+        "tables": {
+            "36": 1,
+        },
+        "split_kw": ["Year", "Years"],
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.BK.A",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆ (1990-2021)"],
+            }
+        },
+    },
+    "bunkers_prec": {
+        "tables": {
+            "37": 0,
+        },
+        "split_kw": "Years",
+        "header": "entity",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.BK.A",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)"],
+            }
+        },
+        "replace_str_data": {",": "."},
+    },
+    "MBIO_CO2": {
+        "dont_read": True,
+        "header": "entity",
+        "tables": {
+            "38": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.BIO",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)"],
+            }
+        },
+    },  # wrong by a factor of 10 until 2004 (compared to inventory and factor)
+    "ffc_gases": {
+        "header": "entity",
+        "tables": {
+            "38": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.A",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%, 2021", "∆(1990-2021)"],
+            }
+        },
+    },
+    "ffc_prec": {
+        "header": "entity",
+        "tables": {
+            "39": 2,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.A",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "СО": "CO",  # looks useless, but necessary as special char present
+                "NOх": "NOx",  # looks useless, but necessary as special char present
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+        "replace_str_data": {",": "."},
+    },
+    "GHG_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "40": 0,
+            "41": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Construction": "M.1.A.2.CON",
+                "Manufacturing industries": "M.1.A.2.IND",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+        "remove_vals": {
+            "r2": {  # GHG sum smaller than CO2 value
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["M.1.A.2.CON"], "time": ["2019", "2020"]},
+            },
+        },
+    },
+    "CO2_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "42": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "CO2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Construction": "M.1.A.2.CON",
+                "Manufacturing industries": "M.1.A.2.IND",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "CH4_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "43": 0,
+            "44": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"CH4 ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Construction": "M.1.A.2.CON",
+                "Manufacturing Industries": "M.1.A.2.IND",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990−2021)", "%, 2021"],
+            }
+        },
+    },
+    "N2O_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "44": 1,
+            "45": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"N2O ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Construction": "M.1.A.2.CON",
+                "Manufacturing Industries": "M.1.A.2.IND",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "CO_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "45": 1,
+            "46": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "CO",
+            "unit": "Gg CO / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Manufacturing Industries and Construction": "1.A.2",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "NOx_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "46": 1,
+            "47": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "NOx",
+            "unit": "Gg NOx / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Manufacturing Industries and Construction": "1.A.2",
+                "Transport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990−2021)", "%, 2021"],
+            }
+        },
+    },
+    "NMVOC_ffc_sectors": {
+        "header": "category",
+        "tables": {
+            "47": 1,
+            "48": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "NMVOC",
+            "unit": "Gg NMVOC / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Manufacturing Industries and Construction": "1.A.2",
+                "Тransport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990−2021)", "%, 2021"],
+            }
+        },
+    },
+    "SO2_ffc_sectors": {
+        "dont_read": True,
+        "header": "category",
+        "tables": {
+            "48": 1,
+            "49": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "SO2",
+            "unit": "t SO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Electricity and Heat Production": "1.A.1",
+                "Manufacturing Industries and Construction": "1.A.2",
+                "Тransport": "1.A.3",
+                "Commercial sector": "1.A.4.a",
+                "Residential sector": "1.A.4.b",
+                "Agriculture": "1.A.4.c",
+                "Total": "1.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990−2021)", "%2021"],
+            }
+        },
+    },  # TODO: inconsistent (parially unit problem?)
+    "fugitive_gases": {
+        "header": "entity",
+        "tables": {
+            "55": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆ (1990-2021)"],
+            }
+        },
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": [f"KYOTOGHG ({gwp_to_use})", "CH4"],
+                "filter": {"time": ["2013"]},
+            }
+        },
+    },
+    "GHG_fugitive_sectors": {
+        "header": "category",
+        "tables": {
+            "56": 0,
+        },
+        "split_kw": "Year",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Coal mining": "1.B.1.a",
+                "Oil": "1.B.2.a",
+                "Natural gas": "1.B.2.b",
+                "Total": "1.B",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%. 2021"],
+            }
+        },
+        "remove_vals": {
+            "2013_error": {
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["1.B.2.b", "1.B"], "time": ["2013"]},
+            },
+        },
+    },
+    "fugitive_prec": {
+        "header": "entity",
+        "tables": {
+            "57": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (2021-1990"],
+            }
+        },
+    },
+    "CH4_coal_sectors": {
+        "header": "category",
+        "tables": {
+            "58": 1,
+            "59": 0,
+        },
+        "ffill_rows": [0],
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"CH4 ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Underground Mining": "1.B.1.a.i.1",
+                "Underground Later stages": "1.B.1.a.i.2",  # not .3 as that's CO2 only and ere we have CH4 only
+                "Underground Sum": "1.B.1.a.i",
+                "Surface Mining": "1.B.1.a.ii.1",
+                "Surface Later stages": "1.B.1.a.ii.2",
+                "Surface Sum": "1.B.1.a.ii",
+                "Total": "1.B.1.a",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)"],
+            },
+        },
+        "rows_to_fix": {2: ["Years"]},
+    },
+    "oil_gases": {  # NMVOC is in Gg
+        "header": "entity",
+        "tables": {
+            "61": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B.2.a",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆ (1990−2021)"],
+            },
+            "f_temp": {
+                "entity": [
+                    "NMVOCs, Gg"
+                ],  # complicated because of different entity. skip for now
+            },
+        },
+    },
+    "GHG_oil_sectors": {
+        "dont_read": True,
+        "header": "category",
+        "tables": {
+            "62": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Oil production": "1.B.2.a.iii.2",
+                "Oil transportation": "1.B.2.a.iii.3",
+                "Total": "1.B.2.a",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990−2021)", "%. 2021"],
+            }
+        },
+    },  # not consistent with inventory (other sector split)
+    "oil_prod_gases": {  # NMVOC is in Gg
+        "dont_read": True,
+        "header": "entity",
+        "tables": {
+            "63": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B.2.a.iii.2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆ (1990-2021)"],
+            },
+            "f_temp": {
+                "entity": [
+                    "NMVOCs, Gg"
+                ],  # complicated because of different entity. skip for now
+            },
+        },
+    },  # venting and flaring summed, not compatble with inventory
+    "oil_trans_gases": {  # NMVOC is in Gg
+        "dont_read": True,
+        "header": "entity",
+        "tables": {
+            "66": 1,
+            "67": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B.2.a.iii.3",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%.2021", "∆ (1990-2021)"],
+            },
+            "f_temp": {
+                "entity": [
+                    "NMVOCs,Gg"
+                ],  # complicated because of different entity. skip for now
+            },
+        },
+    },  # summed with other sectors in inventory
+    "gas_gases": {
+        "header": "entity",
+        "tables": {
+            "69": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B.2.b",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆(1990-2021)"],
+            },
+        },
+        "replace_str_data": {" ": ""},
+    },
+    "gas_prec": {
+        "header": "entity",
+        "tables": {
+            "69": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "1.B.2.b",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+        "replace_str_data": {",": "."},
+    },
+    "IPPU_gases": {
+        "header": "entity",
+        "tables": {
+            "76": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+                "HFCs": f"HFCS ({gwp_to_use})",
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%. 2021", "∆ (1990-2021)"],
+            },
+        },
+    },
+    "IPPU_prec": {
+        "header": "entity",
+        "tables": {
+            "77": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "СО": "CO",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+    },
+    "GHG_IPPU_sectors": {
+        "header": "category",
+        "tables": {
+            "78": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Mineral Industry": "2.A",
+                "Chemical industry": "2.B",
+                "Metal industry": "2.C",
+                "Lubricant use": "2.D.1",
+                "HFCs Use": "2.F",
+                "Total": "2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "CO2_2A_sectors": {
+        "header": "category",
+        "tables": {
+            "81": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "CO2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Cement Production": "2.A.1",
+                "Lime Production": "2.A.2",
+                "Glass Production": "2.A.3",
+                "Ceramics Production": "2.A.4.a",
+                "Total": "2.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "2A1_gases": {
+        "header": "entity",
+        "tables": {
+            "82": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.A.1",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "CO2 emissions from Cement Production": "CO2",
+                "SO2 emissions from Cement Production": "SO2",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            },
+        },
+        "replace_str_data": {",": "."},
+    },
+    "2B_gases": {
+        "header": "entity",
+        "tables": {
+            "92": 0,
+            "93": 0,
+        },
+        "header_remove": [1],
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.B",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СО2,": "CO2",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "N2O,": f"N2O ({gwp_to_use})",
+                "Total GHG": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "", "%, 2021"],
+            },
+            "f2": {"entity": ["CO", "NOx", "SO2", "NMVOCs"]},
+        },
+    },
+    "2B_prec": {
+        "header": "entity",
+        "tables": {
+            "92": 0,
+            "93": 0,
+        },
+        "header_remove": [1],
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.B",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {"unit": "PRIMAP1", "entity": {"NMVOCs": "NMVOC"}},
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "", "%, 2021"],
+            },
+            "f2": {"entity": ["СО2,", "CH4", "N2O,", "Total GHG"]},
+        },
+    },
+    "2C_gases": {
+        "header": "entity",
+        "tables": {
+            "107": 0,
+        },
+        "header_remove": [1],
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.C",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СО2": "CO2",
+                "CH4": f"CH4 ({gwp_to_use})",
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            },
+            "f2": {"entity": ["CO", "NOx", "SO2", "NMVOCs"]},
+        },
+        "rows_to_fix": {-2: ["Years"]},
+    },
+    "2C_prec": {
+        "header": "entity",
+        "tables": {
+            "107": 0,
+        },
+        "header_remove": [1],
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.C",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {"unit": "PRIMAP1", "entity": {"NMVOCs": "NMVOC"}},
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            },
+            "f2": {"entity": ["СО2", "CH4", "Total"]},
+        },
+        "rows_to_fix": {-2: ["Years"]},
+    },
+    "2D_gases": {
+        "header": "entity",
+        "tables": {
+            "115": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.D",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СО2": "CO2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990- 2021)"],
+            },
+        },
+    },
+    "2F_HFCs": {
+        "header": "entity",
+        "tables": {
+            "116": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "2.F.1",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "HFC-32": f"HFC32 ({gwp_to_use})",
+                "HFC-125": f"HFC125 ({gwp_to_use})",
+                "HFC-134a": f"HFC134a ({gwp_to_use})",
+                "HFC-143a": f"HFC143a ({gwp_to_use})",
+                "Total HFCs": f"HFCS ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)"],
+            },
+        },
+    },
+    "2H_NMVOC_sectors": {
+        "header": "category",
+        "tables": {
+            "119": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "NMVOC",
+            "unit": "Gg NMVOC / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Subcategory Alcoholic beverages": "M.2.H.2.a",
+                "Food products": "M.2.H.2.b",
+                "Total": "2.H",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990−2021)", "%. 2021"],
+            },
+        },
+        "rows_to_fix": {2: ["Years"]},
+    },  # TODO: subsectors in IPCC cats?
+    "MAG_gases": {
+        "header": "entity",
+        "tables": {
+            "123": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.AG",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                # "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%, 2021", "∆(1990-2021)"],
+            },
+            "fN2O": {  # indirect N2O from manure missing
+                "entity": ["N2O", "Total"],
+            },
+        },
+    },
+    "GHG_MAG_sectors": {
+        "header": "category",
+        "tables": {
+            "124": 1,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Enteric Fermentation": "3.A.1",
+                "Manure Management": "3.A.2",
+                "Biomass Burning": "M.3.C.1.AG",
+                "N2O Emissions from Managed Soils": "M.AS",  # 3.C.6 included? it's not in Manure management
+                "Rice Cultivation": "3.C.7",
+                # "Total": "M.AG",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%. 2021"],
+            },
+            "ftot": {  # indirect N2O from manure missing
+                "category": ["Total"],
+            },
+        },
+        "remove_vals": {
+            "2020_error": {  # factor of 10 error
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["3.A.2"], "time": ["2020"]},
+            },
+            "mm_error": {  # doesn't fit gas sum
+                "entities": [f"KYOTOGHG ({gwp_to_use})"],
+                "filter": {"category": ["3.A.2"], "time": ["2004", "2005"]},
+            },
+        },
+    },
+    "CH4_3A1_sectors": {
+        "header": "category",
+        "tables": {
+            "127": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"CH4 ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Dairy Cattle": "3.A.1.a.1",
+                "Other Cattle": "3.A.1.a.2",
+                "Sheep": "3.A.1.c",
+                "Goats": "3.A.1.d",
+                "Camels": "3.A.1.e",
+                "Horses": "3.A.1.f",
+                "Mules and Asses": "3.A.1.g",
+                "Swine": "3.A.1.h",
+                "Total": "3.A.1",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "CH4_3A2_sectors": {
+        "header": "category",
+        "tables": {
+            "132": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"CH4 ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Dairy cattle": "3.A.2.a.1",
+                "Other cattle": "3.A.2.a.2",
+                "Sheep": "3.A.2.c",
+                "Goats": "3.A.2.d",
+                "Camels": "3.A.2.e",
+                "Horses": "3.A.2.f",
+                "Mules and Asses": "3.A.2.g",
+                "Swine": "3.A.2.h",
+                "Poultry": "3.A.2.i",
+                "Total": "3.A.2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%,2021"],
+            }
+        },
+    },
+    "N2O_manure_management": {
+        "header": "category",
+        "tables": {
+            "133": 1,
+            "134": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"N2O ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Direct": "3.A.2",
+                "Indirect": "3.C.6",
+                "Total": "M.AG.MM.DirInd",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+    },
+    "N2O_3A2_sectors": {
+        "header": "category",
+        "tables": {
+            "134": 2,
+            "135": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"N2O ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Dairy Cattle": "3.A.2.a.1",
+                "Other Cattle": "3.A.2.a.2",
+                "Dairy cattle": "3.A.2.a.1",
+                "Other cattle": "3.A.2.a.2",
+                "Sheep": "3.A.2.c",
+                "Goats": "3.A.2.d",
+                "Camels": "3.A.2.e",
+                "Horses": "3.A.2.f",
+                "Mules and Asses": "3.A.2.g",
+                "Swine": "3.A.2.h",
+                "Poultry": "3.A.2.i",
+                "Total": "3.A.2",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆ (1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "3C1b_gases": {
+        "header": "entity",
+        "tables": {
+            "137": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "3.C.1.b",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {},
+    },
+    "3C1b_prec": {
+        "header": "entity",
+        "tables": {
+            "138": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "3.C.1.b",
+            "unit": "Gg",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "СО": "CO",
+            },
+        },
+        "filter_remove": {},
+        "replace_str_data": {",": ""},
+    },
+    "N2O_soils": {
+        "header": "category",
+        "tables": {
+            "140": 0,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"N2O ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+            #'unit': 'Gg',
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Direct emissions Synthetic fertilizers": "3.C.4.a",
+                "Organic fertilizers": "3.C.4.b",
+                "Livestock grazing": "3.C.4.c",
+                "Crop residues": "3.C.4.d",
+                "Indirect emissions Atmospheric deposition": "3.C.5.a",
+                "Leaching / runoff": "3.C.5.b",
+                "Total": "M.3.C.45.AG",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            },
+        },
+        "rows_to_fix": {2: ["Years"]},
+    },
+    "CH4_rice": {
+        "header": "entity",
+        "tables": {
+            "144": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "3.C.7",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)"],
+            }
+        },
+    },
+    "CO2_land": {
+        "header": "category",
+        "tables": {
+            "148": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": "CO2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Forest land": "3.B.1",
+                "Cropland": "3.B.2",
+                "Grassland": "3.B.3",
+                "Total": "3.B",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%, 2021"],
+            }
+        },
+    },
+    "forest_fires_gases_prec": {
+        "header": "entity",
+        "tables": {
+            "152": 1,
+        },
+        "long_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "M.FF",  # temp, need to find out where it's included
+            "unit": "t",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "СН4": "CH4",
+                "СО2": "CO2",
+            },
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%, 2021", "∆(1990-2021)"],
+            },
+        },
+        "replace_str_data": {" ": ""},
+    },
+    "waste_gases": {
+        "header": "entity",
+        "tables": {
+            "166": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "4",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["% 2021", "∆(1990-2021)"],
+            },
+        },
+    },
+    "GHG_waste_sectors": {
+        "header": "category",
+        "tables": {
+            "167": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Solid Waste Disposal": "4.A",
+                "Industrial Wastewater": "4.D.2",
+                "Domestic Wastewater": "4.D.1",
+                "Total": "4",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%2021"],
+            },
+            "f2": {
+                "category": [
+                    "Industrial Wastewater",
+                    "Total",
+                ],  # inconsistent with other tables
+            },
+        },
+    },
+    "CH4_solid_waste_sectors": {
+        "header": "category",
+        "tables": {
+            "169": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"CH4 ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Municipal": "M.4.A.Mun",
+                "Industrial": "M.4.A.Ind",
+                "Total": "4.A",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%2021"],
+            }
+        },
+    },
+    "wastewater_gases": {
+        "header": "entity",
+        "tables": {
+            "176": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "4.D",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%2021", "∆(1990-2021)"],
+            },
+        },
+    },
+    "wastewater_sectors": {
+        "header": "category",
+        "tables": {
+            "177": 1,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "category": "category",
+        },
+        "coords_defaults": {
+            "entity": f"KYOTOGHG ({gwp_to_use})",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "category": {
+                "Domestic": "4.D.1",
+                "Industrial": "4.D.2",
+                "Total": "4.D",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["∆(1990-2021)", "%2021"],
+            },
+            "fsec": {  # inconsistent with CH4 data
+                "category": ["Industrial", "Total"],
+            },
+        },
+    },
+    "domestic_ww_gases": {
+        "header": "entity",
+        "tables": {
+            "179": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "4.D.1",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+                "N2O": f"N2O ({gwp_to_use})",
+                "Total": f"KYOTOGHG ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%2021", "∆(1990-2021)"],
+            },
+        },
+    },
+    "industrial_ww_gases": {
+        "header": "entity",
+        "tables": {
+            "183": 1,
+            "184": 0,
+        },
+        "split_kw": "Years",
+        "long_var": "time",
+        "coords_cols": {
+            "entity": "entity",
+        },
+        "coords_defaults": {
+            "category": "4.D.2",
+            "unit": "Gg CO2 / yr",
+        },
+        "coords_value_mapping": {
+            "entity": {
+                "СН4": f"CH4 ({gwp_to_use})",
+            }
+        },
+        "filter_remove": {
+            "f1": {
+                "time": ["%2021", "∆(1990-2021)"],
+            },
+        },
+    },
+}
+
+page_def_trends = {
+    "21": {  # M.0.EL, main gases
+    },
+    "23": {  # main sectors, KYOTOGHG
+        # 'flavor': 'stream',
+        # 'table_areas': ['83,449,563,134'],
+        # 'columns': ['110,155,165,191,214,244,270,301,340,369,401,435,470,503,534'],
+        "flavor": "lattice",
+        "split_text": True,
+    },
+    "24": {  # M.0.EL precursors
+    },
+    "29": {  # Energy, main gases / precursors
+    },
+    "30": {  # Energy sectors, KyotoGHG
+    },
+    "36": {  # bunkers, gases
+    },
+    "37": {  # bunkers, prec
+    },
+    "38": {  # Bio CO2 / 1.A gases
+    },
+    "39": {  # 1.A prec
+    },
+    "40": {  # 1.A sectors, GHG
+    },
+    "41": {  # 1.A sectors, GHG
+    },
+    "42": {  # 1.A sectors, CO2
+    },
+    "43": {  # 1.A sectors, CH4
+    },
+    "44": {  # 1.A sectors, CH4, N2O
+    },
+    "45": {  # 1.A sectors, N2O, CO
+    },
+    "46": {  # 1.A sectors, CO, NOx
+    },
+    "47": {  # 1.A sectors, NOx, NMVOC
+    },
+    "48": {  # 1.A sectors, NMVOC, SO2
+    },
+    "49": {  # 1.A sectors, SO2
+    },
+    "55": {  # 1.B gases
+    },
+    "56": {  # 1.B sectors, GHG
+    },
+    "57": {  # 1.B prec
+    },
+    "58": {  # 1.B.1.a subsectors CH4
+    },
+    "59": {  # 1.B.1.a subsectors CH4
+    },
+    "61": {  # 1.B.2.a gases
+    },
+    "62": {  # 1.B.2.a subsectors GHG
+    },
+    "63": {  # 1.B.2.a.iii.2 gases
+    },
+    "66": {  # 1.B.2.a.iii.3 gases
+    },
+    "67": {  # 1.B.2.a.iii.3 gases
+    },
+    "69": {  # 1.B.2.b gases & prec
+    },
+    "76": {  # IPPU gases
+    },
+    "77": {  # IPPU prec
+    },
+    "78": {  # IPPU subsectors, GHG
+    },
+    "81": {  # 2.A subsectors, CO2
+    },
+    "82": {  # 2.A1, CO2 SO2
+    },
+    "92": {  # 2.B, gases and prec
+        "flavor": "stream",
+        "table_areas": ["37,271,513,113"],
+        "columns": ["85,135,183,244,309,360,411,463"],
+    },
+    "93": {  # 2.B, gases and prec
+        "flavor": "stream",
+        "table_areas": ["83,771,554,427"],
+        "columns": ["122,175,235,294,354,400,450,509"],
+    },
+    "107": {  # 2.C, gases, prec
+        "flavor": "stream",
+        "table_areas": ["88,770,551,305"],
+        "columns": ["129,184,261,318,382,436,504"],
+    },
+    "115": {  # 2.D.1, CO2
+    },
+    "116": {  # 2.F, HFCs
+    },
+    "119": {  # 2.H, NMVOC
+    },
+    "123": {  # M.AG, gases
+    },
+    "124": {  # M.AG sectors, GHG
+    },
+    "127": {  # 3.A.1 subsectors, CH4
+    },
+    "132": {  # 3.A.2 subsectors, CH4
+    },
+    "133": {  # manure management, N2O
+    },
+    "134": {  # manure management, N2O & 3.A.2 subsectors, N2O
+    },
+    "135": {  # 3.A.2 subsectors, N2O
+    },
+    "137": {  # crop residues burning, gases
+    },
+    "138": {  # crop residues burning, prec
+    },
+    "140": {  # soils N2O
+    },
+    "144": {  # rice CH4
+    },
+    "148": {  # land subsectors, CO2
+    },
+    "152": {  # forest fires, gases
+    },
+    "166": {  # waste, gases
+    },
+    "167": {  # waste, sectors
+    },
+    "169": {  # Solid waste, sectors
+    },
+    "176": {  # wastewater, gases
+    },
+    "177": {  # wastewater subsectors, GHG
+    },
+    "179": {  # domestic wastewater, gases
+    },
+    "183": {  # industrial wastewater, gases
+    },
+    "184": {  # industrial wastewater, gases
+    },
+}
+
+table_def_inventory = {
+    "1990_energy": {
+        "time": "1990",
+        "unit": "Gg",
+        "tables": {
+            "187": 0,
+            "188": 0,
+            "189": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {
+            "International Bunkers": "M.BK",
+            "1.A.3.a.i - International Aviation (International Bunkers) (1)": "M.BK.A",
+            "1.A.3.d.i - International water-borne navigation (International bunkers) (1)": "M.BK.M",
+            "1.A.5.c - Multilateral Operations (1)(2)": "M.MULTIOP",
+            "CO2 from Biomass Combustion": "M.BIO",
+            "CO2 from Biomass Combustion Captured": "M.BIOCCS",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "fcat": {"category": ["Memo Items (3)", "Information Items"]},
+        },
+        "remove_vals": {
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "1.A.4.a",
+                        "1.A.4.b",
+                        "1.A.4.c",
+                        "1.A.4",
+                        "1.B",
+                        "1.B.2.a",
+                        "1.B.2.b",
+                        "1.B.2",
+                        "M.BK.A",
+                        "1.A.2",
+                    ]
+                },
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {"category": ["M.BK.A"]},
+            },
+            "SO2_rounding": {  # rounding errors
+                "entities": ["SO2"],
+                "filter": {"category": ["1.A.4", "1.A.4.a", "1.A.4.b", "1.A.4.c"]},
+            },
+        },
+    },
+    "2010_energy": {
+        "time": "2010",
+        "unit": "Gg",
+        "tables": {
+            "189": 1,
+            "190": 0,
+            "191": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {
+            "International Bunkers": "M.BK",
+            "1.A.3.a.i - International Aviation (International Bunkers) (1)": "M.BK.A",
+            "1.A.3.d.i - International water-borne navigation (International bunkers) (1)": "M.BK.M",
+            "1.A.5.c - Multilateral Operations (1)(2)": "M.MULTIOP",
+            "CO2 from Biomass Combustion": "M.BIO",
+            "CO2 from Biomass Combustion Captured": "M.BIOCCS",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "fcat": {"category": ["Memo Items (3)", "Information Items"]},
+        },
+        "remove_vals": {
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "1.A.4.a",
+                        "1.A.4.b",
+                        "1.A.4.c",
+                        "1.A.4",
+                        "1.A.1",
+                        "1.B",
+                        "1.B.2.a",
+                        "1.B.2.b",
+                        "1.B.2",
+                        "M.BK.A",
+                        "1.A.2",
+                    ]
+                },
+            },
+            "NMVOC": {  # reason unclear
+                "entities": ["NMVOC"],
+                "filter": {"category": ["1.A.4.c"]},
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {
+                    "category": [
+                        "1.A.4.c",
+                        "1.B.1.a.i.2",
+                        "1.B.1.a.ii.2",
+                        "M.BK.A",
+                        "1.A.2",
+                    ]
+                },
+            },
+            "SO2_rounding": {  # rounding errors
+                "entities": ["SO2"],
+                "filter": {"category": ["1.A.4", "1.A.4.a", "1.A.4.b", "1.A.4.c"]},
+            },
+        },
+    },
+    "2021_energy": {
+        "time": "2021",
+        "unit": "Gg",
+        "tables": {
+            "191": 1,
+            "192": 0,
+            "193": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {
+            "International Bunkers": "M.BK",
+            "1.A.3.a.i - International Aviation (International Bunkers) (1)": "M.BK.A",
+            "1.A.3.d.i - International water-borne navigation (International bunkers) (1)": "M.BK.M",
+            "1.A.5.c - Multilateral Operations (1)(2)": "M.MULTIOP",
+            "CO2 from Biomass Combustion": "M.BIO",
+            "CO2 from Biomass Combustion Captured": "M.BIOCCS",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {
+            "fcat": {"category": ["Memo Items (3)", "Information Items"]},
+        },
+        "remove_vals": {
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "1.A.4.a",
+                        "1.A.4.b",
+                        "1.A.4.c",
+                        "1.B",
+                        "1.B.2.a",
+                        "1.B.2.b",
+                        "1.B.2",
+                        "M.BK.A",
+                        "1.A.2",
+                    ]
+                },
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {"category": ["1.A.4.c", "M.BK.A"]},
+            },
+        },
+    },
+    "1990_IPPU": {
+        "time": "1990",
+        "tables": {
+            "194": 0,
+            "195": 0,
+            "196": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 1,
+        "manual_repl_unit": {
+            "(Gg)": "Gg",
+            "CO2 Equivalents(Gg)": "GgCO2eq",
+            "N2O": "Gg",
+            "HFCs": "GgCO2eq",
+            "PFCs": "GgCO2eq",
+            "CO2": "Gg",
+            "SF6": "GgCO2eq",
+            "Other halogenated gases with CO2-eq. conversion factors (1)": "GgCO2eq",
+            "NOx": "Gg",
+            "NMVOCs": "Gg",
+            "SO2": "Gg",
+        },
+        "cat_codes_manual": {
+            "2.B.10 - Other (Sulphuric Acid) (3)": "M.2.B.10.a",
+            "2.B.10 - Other (Formaldehyd) (3)": "M.2.B.10.b",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "PFCs": f"PFCS ({gwp_to_use})",
+                "HFCs": f"HFCS ({gwp_to_use})",
+                "NF3": f"NF3 ({gwp_to_use})",
+                "SF6": f"SF6 ({gwp_to_use})",
+                "Other halogenated gases with CO2-eq. conversion factors (1)": f"UnspMixOfHFCs ({gwp_to_use})",
+            },
+        },
+        "filter_remove": {
+            # "fcat": {
+            #     "category": ["Memo Items (3)", "Information Items"]
+            # },
+            "HFCs": {
+                "entity": [
+                    "Other halogenated gases without  CO2-eq. conversion factors (2)",
+                    "Other halogenated gases without CO2-eq. conversion factors (2)",
+                ]
+            }
+        },
+        "ffill_rows": [0],
+        "remove_vals": {
+            "NMVOC_rounding": {  # rounding errors
+                "entities": ["NMVOC"],
+                "filter": {"category": ["2.C"]},
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {"category": ["2", "2.B"]},
+            },
+        },
+    },
+    "2010_IPPU": {
+        "time": "2010",
+        # unit is in header. We need a header def here
+        "tables": {
+            "196": 1,
+            "197": 0,
+            "198": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 1,
+        "manual_repl_unit": {
+            "(Gg)": "Gg",
+            "CO2 Equivalents(Gg)": "GgCO2eq",
+            "N2O": "Gg",
+            "HFCs": "GgCO2eq",
+            "PFCs": "GgCO2eq",
+            "CO2": "Gg",
+            "SF6": "GgCO2eq",
+            "Other halogenated gases with CO2-eq. conversion factors (1)": "GgCO2eq",
+            "NOx": "Gg",
+            "NMVOCs": "Gg",
+            "SO2": "Gg",
+        },
+        "cat_codes_manual": {
+            "2.B.10 - Other (Sulphuric Acid) (3)": "M.2.B.10.a",
+            "2.B.10 - Other (Formaldehyd) (3)": "M.2.B.10.b",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "PFCs": f"PFCS ({gwp_to_use})",
+                "HFCs": f"HFCS ({gwp_to_use})",
+                "NF3": f"NF3 ({gwp_to_use})",
+                "SF6": f"SF6 ({gwp_to_use})",
+                "Other halogenated gases with CO2-eq. conversion factors (1)": f"UnspMixOfHFCs ({gwp_to_use})",
+            },
+        },
+        "filter_remove": {
+            # "fcat": {
+            #     "category": ["Memo Items (3)", "Information Items"]
+            # },
+            "HFCs": {
+                "entity": [
+                    "Other halogenated gases without  CO2-eq. conversion factors (2)",
+                    "Other halogenated gases without CO2-eq. conversion factors (2)",
+                ]
+            }
+        },
+        "ffill_rows": [0],
+        "remove_vals": {
+            "NMVOC_rounding": {  # rounding errors (2.B sum error)
+                "entities": ["NMVOC"],
+                "filter": {"category": ["2.C", "2.B"]},
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {"category": ["2", "2.B"]},
+            },
+        },
+    },
+    "2021_IPPU": {
+        "time": "2021",
+        # 'unit': 'Gg',
+        # unit is in header. We need a header def here
+        "tables": {
+            "199": 0,
+            "200": 0,
+            "201": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 1,
+        "manual_repl_unit": {
+            "(Gg)": "Gg",
+            "CO2 Equivalents(Gg)": "GgCO2eq",
+            "N2O": "Gg",
+            "HFCs": "GgCO2eq",
+            "PFCs": "GgCO2eq",
+            "CO2": "Gg",
+            "SF6": "GgCO2eq",
+            "Other halogenated gases with CO2-eq. conversion factors (1)": "GgCO2eq",
+            "NOx": "Gg",
+            "NMVOCs": "Gg",
+            "SO2": "Gg",
+        },
+        "cat_codes_manual": {
+            "2.B.10 - Other (Sulphuric Acid) (3)": "M.2.B.10.a",
+            "2.B.10 - Other (Formaldehyd) (3)": "M.2.B.10.b",
+        },
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "NMVOCs": "NMVOC",
+                "PFCs": f"PFCS ({gwp_to_use})",
+                "HFCs": f"HFCS ({gwp_to_use})",
+                "NF3": f"NF3 ({gwp_to_use})",
+                "SF6": f"SF6 ({gwp_to_use})",
+                "Other halogenated gases with CO2-eq. conversion factors (1)": f"UnspMixOfHFCs ({gwp_to_use})",
+            },
+        },
+        "filter_remove": {
+            # "fcat": {
+            #     "category": ["Memo Items (3)", "Information Items"]
+            # },
+            "HFCs": {
+                "entity": [
+                    "Other halogenated gases without  CO2-eq. conversion factors (2)",
+                    "Other halogenated gases without CO2-eq. conversion factors (2)",
+                ]
+            }
+        },
+        "ffill_rows": [0],
+        "remove_vals": {
+            "NMVOC_rounding": {  # 2.B sum error
+                "entities": ["NMVOC"],
+                "filter": {"category": ["2.B"]},
+            },
+            "CH4_rounding": {
+                "entities": ["CH4"],
+                "filter": {"category": ["2.C"]},
+            },
+        },
+    },
+    "1990_AFOLU": {
+        "time": "1990",
+        "unit": "Gg",
+        "tables": {
+            "202": 0,
+            "203": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "Net CO2 emissions / removals": "CO2",
+                "CH4": "CH4",
+                "Emissions CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "Emissions NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {},
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": ["CO2"],
+                "filter": {"category": ["3.B.1", "3.B.2", "3.B.3"]},
+            },
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "3.A.2.c",
+                        "3.A.2.d",
+                        "3.A.2.e",
+                        "3.A.2.f",
+                        "3.A.2.g",
+                        "3.A.2.h",
+                        "3.A.2.i",
+                        "3.C.1.b",
+                        "3.C.1",
+                    ]
+                },
+            },
+            "CH4_rounding": {  # rounding errors
+                "entities": ["CH4"],
+                "filter": {
+                    "category": [
+                        "3.A.2.e",
+                        "3.A.2.f",
+                        "3.A.2.g",
+                        "3.C.1",  # sum error
+                    ]
+                },
+            },
+        },
+    },
+    "2010_AFOLU": {
+        "time": "2010",
+        "unit": "Gg",
+        "tables": {
+            "204": 0,
+            "205": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "Net CO2 emissions / removals": "CO2",
+                "CH4": "CH4",
+                "Emissions CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "Emissions NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {},
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": ["NOx", "CO"],
+                "filter": {"category": ["3.B.1"]},
+            },
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "3.A.2.d",
+                        "3.A.2.e",
+                        "3.A.2.f",
+                        "3.A.2.g",
+                        "3.A.2.h",
+                    ]
+                },
+            },
+            "CH4_rounding": {  # rounding errors
+                "entities": ["CH4"],
+                "filter": {
+                    "category": [
+                        "3.A.2.e",
+                        "3.A.1.h",
+                        "3.A.2.h",
+                    ]
+                },
+            },
+        },
+    },
+    "2021_AFOLU": {
+        "time": "2021",
+        "unit": "Gg",
+        "tables": {
+            "206": 0,
+            "207": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "Net CO2 emissions / removals": "CO2",
+                "CH4": "CH4",
+                "Emissions CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "Emissions NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+            },
+        },
+        "filter_remove": {},
+        "remove_vals": {
+            "r1": {  # value inconsistent with fugitive table
+                "entities": ["NOx", "CO"],
+                "filter": {"category": ["3.B.1"]},
+            },
+            "N2O_rounding": {  # rounding errors
+                "entities": ["N2O"],
+                "filter": {
+                    "category": [
+                        "3.A.2.d",
+                        "3.A.2.e",
+                        "3.A.2.f",
+                        "3.A.2.g",
+                        "3.A.2.h",
+                    ]
+                },
+            },
+            "CH4_rounding": {  # rounding errors
+                "entities": ["CH4"],
+                "filter": {"category": ["3.A.1.h", "3.A.2.e", "3.A.2.g", "3.A.2.h"]},
+            },
+        },
+    },
+    "1990_waste": {
+        "time": "1990",
+        "unit": "Gg",
+        "tables": {
+            "208": 0,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "CO2": "CO2",
+                "CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+                "SO2": "SO2",
+            },
+        },
+        "filter_remove": {},
+    },
+    "2010_waste": {
+        "time": "2010",
+        "unit": "Gg",
+        "tables": {
+            "208": 1,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "CO2": "CO2",
+                "CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+                "SO2": "SO2",
+            },
+        },
+        "filter_remove": {},
+    },
+    "2021_waste": {
+        "time": "2021",
+        "unit": "Gg",
+        "tables": {
+            "208": 2,
+        },
+        "unit_row": 0,
+        "entity_row": 0,
+        "cat_codes_manual": {},
+        "coords_value_mapping": {
+            "unit": "PRIMAP1",
+            "entity": {
+                "CO2": "CO2",
+                "CH4": "CH4",
+                "N2O": "N2O",
+                "NOx": "NOx",
+                "CO": "CO",
+                "NMVOCs": "NMVOC",
+                "SO2": "SO2",
+            },
+        },
+        "filter_remove": {},
+    },
+}
+
+page_def_inventory = {
+    "187": {  # 1990, energy 1
+        # unit: Gg
+    },
+    "188": {  # 1990, energy 2
+        # unit: Gg
+    },
+    "189": {  # 1990, energy 3 / 2010, energy 1
+        # unit: Gg
+    },
+    "190": {  # 2010, energy 2
+        # unit: Gg
+    },
+    "191": {  # 2010, energy 3 / 2021, energy 1
+        "split_text": True,
+        # unit: Gg
+    },
+    "192": {  # 2021, energy 2
+        # unit: Gg
+    },
+    "193": {  # 2021, energy 3
+        # unit: Gg
+    },
+    "194": {  # 1990, IPPU 1
+        # unit: header
+    },
+    "195": {  # 1990, IPPU 2
+        "flavor": "stream",
+        "table_areas": ["50,513,789,56"],
+        "columns": ["287,323,355,383,410,438,466,495,566,638,679,714,754"],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            }
+        },
+    },
+    "196": {  # 1990, IPPU 3 / 2010, IPPU 1
+        "flavor": "stream",
+        "table_areas": ["50,513,789,330", "50,303,789,50"],
+        "columns": [
+            "287,323,355,383,410,438,466,495,566,638,679,714,754",
+            "287,323,355,383,410,438,466,495,566,638,679,714,754",
+        ],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            },
+            1: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            },
+        },
+    },
+    "197": {  # 2010, IPPU 2
+        "flavor": "stream",
+        "table_areas": ["50,513,789,56"],
+        "columns": ["287,323,355,383,410,438,466,495,566,638,679,714,754"],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            }
+        },
+    },
+    "198": {  # 2010, IPPU 3
+        "flavor": "stream",
+        "table_areas": ["50,513,789,153"],
+        "columns": ["287,323,355,383,410,438,466,495,566,638,679,714,754"],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            }
+        },
+    },
+    "199": {  # 2021, IPPU 1
+        "flavor": "lattice",
+        # unit: header
+    },
+    "200": {  # 2021, IPPU 2
+        "flavor": "stream",
+        "table_areas": ["50,513,789,53"],
+        "columns": ["287,323,355,383,410,438,466,495,566,638,679,714,754"],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            }
+        },
+    },
+    "201": {  # 2010, IPPU 3
+        "flavor": "stream",
+        "table_areas": ["50,513,789,320"],
+        "columns": ["287,323,355,383,410,438,466,495,566,638,679,714,754"],
+        # unit: header
+        "table_config": {
+            0: {
+                "rows_to_fix": {-5: ["Categories"]},
+                "bfill_header": [0, 1],
+            }
+        },
+    },
+    "202": {  # 1990, AFOLU 1
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {2: ["Categories"]},
+            }
+        },
+    },
+    "203": {  # 1990, AFOLU 2
+        "flavor": "stream",
+        "table_areas": ["81,787,556,233"],
+        "columns": ["291,353,394,433,475,515"],
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {-4: ["Categories"]},
+            }
+        },
+    },
+    "204": {  # 2010, AFOLU 1
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {2: ["Categories"]},
+            }
+        },
+    },
+    "205": {  # 2010, AFOLU 2
+        "flavor": "stream",
+        "table_areas": ["81,787,556,233"],
+        "columns": ["291,353,394,433,475,515"],
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {-4: ["Categories"]},
+            }
+        },
+    },
+    "206": {  # 2021, AFOLU 1
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {2: ["Categories"]},
+            }
+        },
+    },
+    "207": {  # 2021, AFOLU 2
+        "flavor": "stream",
+        "table_areas": ["81,787,556,233"],
+        "columns": ["291,353,394,433,475,515"],
+        # unit: Gg
+        "table_config": {
+            0: {
+                "rows_to_fix": {-4: ["Categories"]},
+            }
+        },
+    },
+    "208": {  # Waste, 1990, 2010, 2021
+        # unit: Gg
+    },
+}
+
+config_general = {
+    "time_format": "%Y",
+    "coords_defaults": {
+        "area": "UZB",
+        "source": "UZB-GHG-Inventory",
+        "provenance": "measured",
+        "scenario": "BTR1",
+    },
+    "coords_terminologies": {
+        "area": "ISO3",
+        "category": "IPCC2006_PRIMAP",
+        "scenario": "PRIMAP",
+    },
+    "meta_data": {
+        "rights": "",
+        "references": "https://unfccc.int/documents/640099",
+        "contact": "mail@johannes-guetschow.de",
+        "title": "NATIONAL REPORT - Inventory of anthropogenic emissions sources and sinks of greenhouse gases in the Republic of Uzbekistan 1990-2021",
+        "comment": "Read fom pdf file by Johannes Gütschow",
+        "institution": "United Nations Framework Convention on Climate Change (UNFCCC)",
+    },
+}
+
+
+aggregate_coords = {
+    f"category ({config_general['coords_terminologies']['category']})": {
+        "1.A.2": {
+            "sources": ["M.1.A.2.CON", "M.1.A.2.IND"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O", f"KYOTOGHG ({gwp_to_use})"],
+            },
+        },
+        "1.A.4": {
+            "sources": ["1.A.4.a", "1.A.4.b", "1.A.4.c"],
+            "filter": {
+                "entity": [
+                    "CH4",
+                    "CO2",
+                    "N2O",
+                    f"KYOTOGHG ({gwp_to_use})",
+                    "SO2",
+                    "NMVOC",
+                    "CO",
+                    "NOx",
+                ],
+            },
+        },
+        "1.B.1": {
+            "sources": ["1.B.1.a"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "1.B.2": {
+            "sources": ["1.B.2.a", "1.B.2.b"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "1.B": {
+            "sources": ["1.B.1", "1.B.2", "1.B.3"],
+            "filter": {
+                "entity": ["CH4"],
+                "time": ["2013"],
+            },
+        },
+        "1": {
+            "sources": ["1.A", "1.B", "1.C"],
+            "filter": {
+                "entity": ["CH4"],
+                "time": ["2013"],
+            },
+        },
+        "2.F": {
+            "sources": ["2.F.1"],
+            "filter": {
+                "entity": ["HFC125", "HFC134a", "HFC143a", "HFC32"],
+            },
+        },
+        "2": {
+            "sources": ["2.F"],
+            "filter": {
+                "entity": ["HFC125", "HFC134a", "HFC143a", "HFC32"],
+            },
+        },
+        "3.A": {
+            "sources": ["3.A.1", "3.A.2"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "3.C.1": {
+            "sources": ["3.C.1.a", "3.C.1.b"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.3.C.1.AG": {
+            "sources": ["3.C.1.b"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.3.C.1.LU": {
+            "sources": ["3.C.1.a"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        # "3.C": {"sources": [
+        #     "3.C.1", "3.C.4", "3.C.5", "3.C.6", "3.C.7", "3.C.8", "3.C.9",
+        #     "3.C.10", "3.C.11", "3.C.12", "3.C.13"]},
+        "3.C": {
+            "sources": [  # build from available time-series not inventory
+                "3.C.1",
+                "M.3.C.45.AG",
+                "3.C.6",
+                "3.C.7",
+            ],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.3.C.AG": {
+            "sources": [  # build from available time-series not inventory
+                "M.3.C.1.AG",
+                "M.3.C.45.AG",
+                "3.C.6",
+                "3.C.7",
+            ],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.AG.ELV": {
+            "sources": ["M.3.C.AG"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.AG": {  # consistency check
+            "sources": ["M.AG.ELV", "3.A"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "M.LULUCF": {
+            "sources": ["3.B", "3.D", "M.3.C.1.LU"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O"],
+            },
+        },
+        "3": {
+            "sources": ["M.AG", "M.LULUCF"],
+            "filter": {
+                "entity": ["CH4", "CO2", "N2O", f"KYOTOGHG ({gwp_to_use})"],
+            },
+        },
+        "0": {  # consistency check
+            "sources": ["1", "2", "3", "4"],
+            "filter": {
+                "entity": [
+                    "CH4",
+                    "CO2",
+                    "N2O",
+                    "HFC125",
+                    "HFC134a",
+                    "HFC143a",
+                    "HFC32",
+                    f"UnspMixOfHFCs ({gwp_to_use})",
+                ],
+            },
+        },
+        "M.0.EL": {  # consistency check
+            "sources": ["1", "2", "M.AG", "4"],
+            "filter": {
+                "entity": [
+                    "CH4",
+                    "CO2",
+                    "N2O",
+                    "HFC125",
+                    "HFC134a",
+                    "HFC143a",
+                    "HFC32",
+                    f"UnspMixOfHFCs ({gwp_to_use})",
+                ],
+            },
+        },
+    }
+}
+
+basket_copy = {
+    "GWPs_to_add": ["SARGWP100", "AR5GWP100", "AR6GWP100"],
+    "entities": ["UnspMixOfHFCs", "PFCS"],
+    "source_GWP": gwp_to_use,
+}
+
+gas_baskets = {
+    "HFCS (SARGWP100)": [
+        "HFC125",
+        "HFC134a",
+        "HFC143a",
+        "HFC32",
+        "UnspMixOfHFCs (SARGWP100)",
+    ],
+    "HFCS (AR4GWP100)": [
+        "HFC125",
+        "HFC134a",
+        "HFC143a",
+        "HFC32",
+        "UnspMixOfHFCs (AR4GWP100)",
+    ],
+    "HFCS (AR5GWP100)": [
+        "HFC125",
+        "HFC134a",
+        "HFC143a",
+        "HFC32",
+        "UnspMixOfHFCs (AR5GWP100)",
+    ],
+    "HFCS (AR6GWP100)": [
+        "HFC125",
+        "HFC134a",
+        "HFC143a",
+        "HFC32",
+        "UnspMixOfHFCs (AR6GWP100)",
+    ],
+    "FGASES (SARGWP100)": ["HFCS (SARGWP100)", "PFCS (SARGWP100)", "SF6", "NF3"],
+    "FGASES (AR4GWP100)": ["HFCS (AR4GWP100)", "PFCS (AR4GWP100)", "SF6", "NF3"],
+    "FGASES (AR5GWP100)": ["HFCS (AR5GWP100)", "PFCS (AR5GWP100)", "SF6", "NF3"],
+    "FGASES (AR6GWP100)": ["HFCS (AR6GWP100)", "PFCS (AR6GWP100)", "SF6", "NF3"],
+    "KYOTOGHG (SARGWP100)": ["CO2", "CH4", "N2O", "FGASES (SARGWP100)"],
+    "KYOTOGHG (AR4GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR4GWP100)"],
+    "KYOTOGHG (AR5GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR5GWP100)"],
+    "KYOTOGHG (AR6GWP100)": ["CO2", "CH4", "N2O", "FGASES (AR6GWP100)"],
+}

src/unfccc_ghg_data/unfccc_reader/Uzbekistan/read_UZB_BTR1_from_pdf.py

@@ -0,0 +1,480 @@
+"""
+Read Uzbekistans's BTR1 inventory from pdf
+
+Most tables are read, but not all.
+Some tables are ignored due to inconsistencies (precursors)
+
+
+"""
+import camelot
+import numpy as np
+import pandas as pd
+import primap2 as pm2
+
+from unfccc_ghg_data.helper import (
+    downloaded_data_path,
+    extracted_data_path,
+    fix_rows,
+    make_long_table,
+    process_data_for_country,
+    set_to_nan_in_ds,
+)
+from unfccc_ghg_data.unfccc_reader.Uzbekistan.config_uzb_btr1 import (
+    aggregate_coords,
+    basket_copy,
+    cat_code_regexp,
+    config_general,
+    gas_baskets,
+    page_def_inventory,
+    page_def_trends,
+    table_def_inventory,
+    table_def_trends,
+)
+
+if __name__ == "__main__":
+    pd.set_option("future.no_silent_downcasting", True)
+
+    # ###
+    # configuration
+    # ###
+
+    # folders and files
+    input_folder = downloaded_data_path / "UNFCCC" / "Uzbekistan" / "BTR1"
+    output_folder = extracted_data_path / "UNFCCC" / "Uzbekistan"
+    if not output_folder.exists():
+        output_folder.mkdir()
+
+    output_filename = "UZB_BTR1NIR_"
+
+    inventory_file = "NIR_Uzb_eng_26_06_2024_%D1%84%D0%B8%D0%BD%D0%B0%D0%BB.pdf"
+
+    compression = dict(zlib=True, complevel=9)
+
+    # ###
+    # start data reading
+    # ###
+
+    exceptions = {}
+    # trend tables
+    data_trend = None
+    tables_to_read = table_def_trends.keys()
+    # tables_to_read = ["bunkers_gases"]
+
+    for table in tables_to_read:
+        table_def_current = table_def_trends[table].copy()
+        if "dont_read" in table_def_current.keys():
+            if table_def_current["dont_read"] is True:
+                print(f"Skipping table {table}")
+                continue
+
+        print(f"Reading {table}")
+
+        data_table = None
+        for page in table_def_current["tables"]:
+            # prep parameters
+            if "rows_to_fix" in table_def_current:
+                rows_to_fix = table_def_current["rows_to_fix"]
+            else:
+                rows_to_fix = {}
+            # read the page
+            tables_read = camelot.read_pdf(
+                str(input_folder / inventory_file), pages=page, **page_def_trends[page]
+            )
+            table_df = tables_read[table_def_current["tables"][page]].df
+
+            # fill empty strings in rows
+            if "ffill_rows" in table_def_current.keys():
+                for row in table_def_current["ffill_rows"]:
+                    table_df.iloc[row] = (
+                        table_df.iloc[row].replace(r"^\s*$", np.nan, regex=True).ffill()
+                    )
+
+            # fix split rows
+            if rows_to_fix:
+                for n_rows in rows_to_fix.keys():
+                    table_df = fix_rows(table_df, rows_to_fix[n_rows], 0, n_rows)
+
+            # remove additional header rows
+            if "header_remove" in table_def_current.keys():
+                idx_to_drop = table_df.index[table_def_current["header_remove"]]
+                table_df = table_df.drop(idx_to_drop)
+
+            # remove unwanted characters in header
+            table_df.iloc[0] = table_df.iloc[0].str.replace("\n", " ")
+            table_df.iloc[0] = table_df.iloc[0].str.replace("  ", " ")
+            table_df.iloc[0] = table_df.iloc[0].str.strip()
+
+            # stack table if necessary
+            if "split_kw" in table_def_current:
+                split_kw = table_def_current["split_kw"]
+                if isinstance(split_kw, list):
+                    for kw in split_kw[1:]:
+                        table_df.iloc[0] = table_df.iloc[0].str.replace(kw, split_kw[0])
+                    long_kw = split_kw[0]
+                else:
+                    long_kw = split_kw
+                table_df = make_long_table(table_df, keyword=long_kw)
+
+            else:
+                long_kw = table_def_current["long_kw"]
+                table_df.columns = table_df.iloc[0]
+                idx_to_drop = table_df.index[0]
+                table_df = table_df.drop(idx_to_drop)
+
+            # remove unwanted characters in first column
+            table_df[long_kw] = table_df[long_kw].str.replace("м", "")
+            table_df[long_kw] = table_df[long_kw].str.replace("\n", " ")
+            table_df[long_kw] = table_df[long_kw].str.replace("  ", " ")
+
+            # stack header variable
+            table_df.index = table_df[long_kw]
+            table_df = table_df.drop(columns=[long_kw])
+            table_df.columns.name = table_def_current["header"]
+            table_long = table_df.stack()  # noqa: PD013
+            table_long.name = "data"
+            table_long = table_long.reset_index()
+            table_long = table_long.rename(
+                columns={long_kw: table_def_current["long_var"]}
+            )
+
+            # replace str in data
+            if "replace_str_data" in table_def_current.keys():
+                for str_to_replace in table_def_current["replace_str_data"].keys():
+                    table_long["data"] = table_long["data"].str.replace(
+                        str_to_replace,
+                        table_def_current["replace_str_data"][str_to_replace],
+                    )
+
+            # prepare parameters for conversion to PRIMAP2 IF
+            coords_defaults = config_general["coords_defaults"].copy()
+            coords_defaults.update(table_def_current["coords_defaults"])
+
+            data_if = pm2.pm2io.convert_long_dataframe_if(
+                table_long,
+                coords_cols=table_def_current["coords_cols"],
+                coords_defaults=coords_defaults,
+                coords_terminologies=config_general["coords_terminologies"],
+                coords_value_mapping=table_def_current["coords_value_mapping"],
+                filter_remove=table_def_current["filter_remove"],
+                meta_data=config_general["meta_data"],
+                time_format=config_general["time_format"],
+            )
+            data_pm2 = pm2.pm2io.from_interchange_format(data_if)
+            if data_table is None:
+                data_table = data_pm2
+            else:
+                data_table = data_table.pr.merge(data_pm2)
+
+        if "remove_vals" in table_def_current:
+            for case in table_def_current["remove_vals"]:
+                data_table = set_to_nan_in_ds(
+                    data_table, **table_def_current["remove_vals"][case]
+                )
+
+        if data_trend is None:
+            data_trend = data_table
+        else:
+            data_trend = data_trend.pr.merge(data_table)
+
+    # inventories for 1990, 2010, 2021
+    data_inv = None
+    tables_to_read = table_def_inventory.keys()
+
+    def repl(m):  # noqa: D103
+        return m.group("code")
+
+    for table in tables_to_read:
+        print(f"Reading {table}")
+        table_def_current = table_def_inventory[table].copy()
+        data_table = None
+        for page in table_def_current["tables"]:
+            table_number = table_def_current["tables"][page]
+            print(f"page {page}, {table_number}")
+            # prep
+            if "manual_repl_unit" in table_def_current:
+                manual_repl_unit = table_def_current["manual_repl_unit"]
+            else:
+                manual_repl_unit = None
+
+            if "unit" in table_def_current.keys():
+                default_unit = table_def_current["unit"]
+            else:
+                default_unit = "Gg"
+
+            page_def = page_def_inventory[page].copy()
+            if "table_config" in page_def.keys():
+                table_config = page_def.pop("table_config")[table_number]
+            else:
+                table_config = {}
+
+            if "rows_to_fix" in table_config:
+                rows_to_fix = table_config["rows_to_fix"]
+            else:
+                rows_to_fix = None
+
+            if "bfill_header" in table_config:
+                bfill_header = table_config["bfill_header"]
+            else:
+                bfill_header = None
+
+            # read the page
+            tables_read = camelot.read_pdf(
+                str(input_folder / inventory_file), pages=page, **page_def
+            )
+            table_df = tables_read[table_def_current["tables"][page]].df
+
+            # fix split rows if necessary
+            if rows_to_fix is not None:
+                for n_rows in rows_to_fix.keys():
+                    table_df = fix_rows(table_df, rows_to_fix[n_rows], 0, n_rows)
+
+            if bfill_header is not None:
+                table_df.iloc[bfill_header] = (
+                    table_df.iloc[bfill_header]
+                    .replace(r"^\s*$", np.nan, regex=True)
+                    .bfill()
+                )
+
+                # remove unwanted characters in header
+            table_df.iloc[table_def_current["unit_row"]] = table_df.iloc[
+                table_def_current["unit_row"]
+            ].str.replace("\n", " ")
+            table_df.iloc[table_def_current["unit_row"]] = table_df.iloc[
+                table_def_current["unit_row"]
+            ].str.replace("  ", " ")
+            table_df.iloc[table_def_current["unit_row"]] = table_df.iloc[
+                table_def_current["unit_row"]
+            ].str.strip()
+            table_df.iloc[table_def_current["entity_row"]] = table_df.iloc[
+                table_def_current["entity_row"]
+            ].str.replace("\n", " ")
+            table_df.iloc[table_def_current["entity_row"]] = table_df.iloc[
+                table_def_current["entity_row"]
+            ].str.replace("  ", " ")
+            table_df.iloc[table_def_current["entity_row"]] = table_df.iloc[
+                table_def_current["entity_row"]
+            ].str.strip()
+
+            # fill empty strings in rows
+            if "ffill_rows" in table_def_current.keys():
+                for row in table_def_current["ffill_rows"]:
+                    table_df.iloc[row] = (
+                        table_df.iloc[row].replace(r"^\s*$", np.nan, regex=True).ffill()
+                    )
+
+            table_df = pm2.pm2io.nir_add_unit_information(
+                table_df,
+                unit_row=table_def_current["unit_row"],
+                entity_row=table_def_current["entity_row"],
+                regexp_entity=".*",
+                default_unit=default_unit,
+                manual_repl_unit=manual_repl_unit,
+            )
+            table_df = table_df.set_index(table_df.columns[0])
+            table_long = pm2.pm2io.nir_convert_df_to_long(
+                table_df,
+                year=table_def_current["time"],
+                header_long=["category", "entity", "unit", "time", "data"],
+            )
+
+            # replace special characters in category column
+            table_long["category"] = table_long["category"].str.replace("\n", " ")
+            table_long["category"] = table_long["category"].str.replace("  ", " ")
+            table_long["category"] = table_long["category"].str.strip()
+
+            # replace cat names by codes in col "category"
+            # first the manual replacements
+            table_long["category"] = table_long["category"].replace(
+                table_def_current["cat_codes_manual"]
+            )
+
+            # then the regex replacements
+            table_long["category"] = table_long["category"].str.replace(
+                cat_code_regexp, repl, regex=True
+            )
+
+            # # prepare parameters for conversion to PRIMAP2 IF
+            coords_defaults = config_general["coords_defaults"].copy()
+            # coords_defaults.update(table_def_current['coords_defaults'])
+            coords_cols = {
+                "category": "category",
+                "entity": "entity",
+                "unit": "unit",
+            }
+
+            data_if = pm2.pm2io.convert_long_dataframe_if(
+                table_long,
+                coords_cols=coords_cols,
+                coords_defaults=coords_defaults,
+                coords_terminologies=config_general["coords_terminologies"],
+                coords_value_mapping=table_def_current["coords_value_mapping"],
+                filter_remove=table_def_current["filter_remove"],
+                meta_data=config_general["meta_data"],
+                time_format=config_general["time_format"],
+            )
+            data_pm2 = pm2.pm2io.from_interchange_format(data_if)
+            if data_table is None:
+                data_table = data_pm2
+            else:
+                data_table = data_table.pr.merge(data_pm2)
+
+        if "remove_vals" in table_def_current:
+            for case in table_def_current["remove_vals"]:
+                data_table = set_to_nan_in_ds(
+                    data_table, **table_def_current["remove_vals"][case]
+                )
+
+        if data_inv is None:
+            data_inv = data_table
+        else:
+            data_inv = data_inv.pr.merge(data_table)
+
+    # remove SO2, NOx from trends as there are inconsistencies with the inventory
+    # SO2: 2, 2.B, 2.C, 1.A
+    # NOx: 1.A.3, 2.C
+    vars_to_remove = ["SO2", "NOx"]
+    vars_trends = data_trend.data_vars
+    vars_to_keep = [var for var in vars_trends if var not in vars_to_remove]
+    data_trend = data_trend[vars_to_keep]
+
+    # merge trends and inventory
+    data_pm2 = data_trend.pr.merge(data_inv)
+
+    # convert back to IF to have units in the fixed format
+    data_if = data_pm2.pr.to_interchange_format()
+
+    # ###
+    # save data to IF and native format
+    # ###
+    pm2.pm2io.write_interchange_format(
+        output_folder
+        / (
+            output_filename
+            + config_general["coords_terminologies"]["category"]
+            + "_raw"
+        ),
+        data_if,
+    )
+
+    encoding = {var: compression for var in data_pm2.data_vars}
+    data_pm2.pr.to_netcdf(
+        output_folder
+        / (
+            output_filename
+            + config_general["coords_terminologies"]["category"]
+            + "_raw"
+            + ".nc"
+        ),
+        encoding=encoding,
+    )
+
+    # ###
+    # sector aggregation
+    # ###
+
+    data_pm2_agg = data_pm2.copy()
+
+    # actual processing
+
+    country_processing = {
+        "basket_copy": basket_copy,
+        "aggregate_coords": aggregate_coords,
+    }
+
+    data_pm2_agg = process_data_for_country(
+        data_pm2_agg,
+        entities_to_ignore=[],
+        gas_baskets=gas_baskets,
+        processing_info_country=country_processing,
+        # sectors_out=sectors_to_save,
+    )
+
+    # adapt source and metadata
+    current_source = data_pm2_agg.coords["source"].to_numpy()[0]
+    data_temp = data_pm2_agg.pr.loc[{"source": current_source}]
+    data_pm2_agg = data_pm2_agg.pr.set("source", "AI_INV", data_temp)
+    data_pm2_agg = data_pm2_agg.pr.loc[{"source": ["AI_INV"]}]
+
+    # convert back to IF to have units in the fixed format
+    data_if_agg = data_pm2_agg.pr.to_interchange_format()
+
+    pm2.pm2io.write_interchange_format(
+        output_folder
+        / (output_filename + config_general["coords_terminologies"]["category"]),
+        data_if_agg,
+    )
+
+    encoding = {var: compression for var in data_pm2_agg.data_vars}
+    data_pm2_agg.pr.to_netcdf(
+        output_folder
+        / (
+            output_filename + config_general["coords_terminologies"]["category"] + ".nc"
+        ),
+        encoding=encoding,
+    )
+
+
+# NIR tables for 1990, 2010, 2021 on page 187-208
+# trend tables:
+# precursors not listed, but also available
+# mot all detail tables listed
+# [X] Total for main gases: 21
+# [X] KyotoGHG for main sectors: 23
+# [X] gases for energy sector totals: 29
+# [X] Kyotoghg for energy sector main subsectors: 30
+# [X] Bunkers, gases: 36
+# [X] 1.A gases: 38
+# [X] KyotoGHG for main sectors: 40, 41
+# [X] CO2 for 1.A.X sectors: 42
+# [X] CH4 for 1.A.X sectors: 43,44
+# [X] N2O for 1.A.X sectors: 44, 45
+# [] KyotoGHG 1.A.3 subsectors: 51
+# [X] 1.B gases: 55
+# [X] 1.B.x KyotoGHG: 56
+# [x] CH4 from coal mining: 58, 59 (check if that covers 1.B.1)
+# [x] gases from oil (1.B.2.a?): 61
+# [X] KyotoGHG for oil subsectors: 62
+# [X] gases from oil production: 63
+# [] KyotoGHG oil production subsectors: 64
+# [X] gases from oil transportation: 66, 67
+# [x] gases for natural gas: 69
+# [x] precursors for natural gas: 69
+# [] KyotoGHG for natural gas subsectors
+# [X] IPPU gases: 76
+# [X] IPPU precursors: 76
+# [X] KyotoGHG IPPU subsectors: 78
+# [x] CO2 2.A: 81
+# [X] CO2, SO2 in 2.A.1
+# [] Skip the other 2.A.x tables as contained in table on page 81
+# [X] 2.B gases: 92, 93
+# [] KyotoGHG 2.B.x: 94
+# [] Detail tables for individual chemicals (95, 98, 101, 103, 105)
+# [X] 2.C gases: 107
+# [] Metal industry detail tables (108, 109, 111, 112, 113)
+# [X] 2.D.1 CO2: 115 (other sectors NE)
+# [X] 2.F gases: 116
+# [X] 2.H NMVOC only
+##########################
+# [X] M.AG, gases: 123
+# [X] KyotoGHG M.AG subcategories: 124
+# [X] CH4 3.A.1.x : 127
+# [] manure management gases: 131 (not needed)
+# [X] 3.A.2.x, CH4: 132
+# [X] direct + indirect N2O manure management: 133, 134
+# [X] direct N2O mm, subsectors: 134, 135
+# [X] crop residues burning gases: 137 (all other burning NO)
+# [] Liming: NO, Urea: NO, but text reads like NE
+# [X] N2O from soils: 140
+# [X] CH4 from rice: 144
+# [X] KyotoGHG Land+subsectors: 148 (CO2 only)
+# [] CO2 removals in Forest land remaining forest land (fires included): 151
+# [X] gases for forest fires (read for non-CO2): 152
+# [] CO2 from cropland remaining cropland: 158
+# [] CO2 from grassland remaining grassland: 163
+# [X] Waste by gas: 166
+# [X] Waste by sector: 167
+# [X] CH4 from solid waste: 169
+# [X] Wastewater CH4, N2O: 176
+# [X] Wastewater KyotoGHG for subsectors: 177
+# [X] Domestic wastewater, gases: 179
+# [X] Industrial wastewater CH$: 183

src/unfccc_ghg_data/unfccc_reader/folder_mapping.json

@@ -12,6 +12,7 @@
     "IDN": "Indonesia",
     "ISR": "Israel",
     "JPN": "Japan",
+    "KNA": "Saint_Kitts_and_Nevis",
     "KOR": "Republic_of_Korea",
     "MAR": "Morocco",
     "MEX": "Mexico",
@@ -24,5 +25,6 @@
     "SGP": "Singapore",
     "THA": "Thailand",
     "TWN": "Taiwan",
-    "USA": "United_States_of_America"
+    "USA": "United_States_of_America",
+    "UZB": "Uzbekistan"
 }