From be0954c1211b6f7b54f8a2460b7967fc0a853613 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 18 Jun 2021 17:18:13 -0400 Subject: [PATCH 01/25] Add exampele3 with pynautobot --- examples/example3/README.md | 32 + examples/example3/countries.json | 1626 +++++++++++++++++++++++++ examples/example3/local_adapter.py | 55 + examples/example3/main.py | 47 + examples/example3/models.py | 34 + examples/example3/nautobot_adapter.py | 167 +++ examples/example3/requirements.txt | 2 + 7 files changed, 1963 insertions(+) create mode 100644 examples/example3/README.md create mode 100644 examples/example3/countries.json create mode 100644 examples/example3/local_adapter.py create mode 100644 examples/example3/main.py create mode 100644 examples/example3/models.py create mode 100644 examples/example3/nautobot_adapter.py create mode 100644 examples/example3/requirements.txt diff --git a/examples/example3/README.md b/examples/example3/README.md new file mode 100644 index 0000000..bf668c6 --- /dev/null +++ b/examples/example3/README.md @@ -0,0 +1,32 @@ + +# Example 3 + +This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like via a REST API like Nautobot. + +For this example, we have a shared model for Region and Country defined in `models.py`. +A Country must be associated with a Region and can be part of a Subregion too. + +The comparison and synchronization of dataset is done between a local JSON file and the [public instance of Nautobot](https://demo.nautobot.com). + + +## Install the requirements + +to use this example you must have some dependencies installed, please make sure to run +``` +pip install -r requirements.txt +``` + +## Try the example + +The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define. +After the first sync, the diff should show no difference. +At this point, Diffsync will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode. + +``` +### DIFF Compare the data between Nautobot and the local JSON file. +main.py --diff + +### SYNC Update the list of country in Nautobot. +main.py --sync +``` + diff --git a/examples/example3/countries.json b/examples/example3/countries.json new file mode 100644 index 0000000..8828a0e --- /dev/null +++ b/examples/example3/countries.json @@ -0,0 +1,1626 @@ +[ + { + "country": "Vatican City", + "pop2021": "0.8000", + "area": 1, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Tokelau", + "pop2021": "1.3730", + "area": 12, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Niue", + "pop2021": "1.6190", + "area": 260, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Falkland Islands", + "pop2021": "3.5330", + "area": 12173, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Montserrat", + "pop2021": "4.9770", + "area": 102, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Saint Pierre and Miquelon", + "pop2021": "5.7660", + "area": 242, + "region": "Americas", + "subregion": "Northern America" + }, + { + "country": "Saint Barthelemy", + "pop2021": "9.9070", + "area": 21, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Nauru", + "pop2021": "10.8760", + "area": 21, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Wallis and Futuna", + "pop2021": "11.0940", + "area": 142, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Tuvalu", + "pop2021": "11.9310", + "area": 26, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Anguilla", + "pop2021": "15.1170", + "area": 91, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Cook Islands", + "pop2021": "17.5650", + "area": 236, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Palau", + "pop2021": "18.1690", + "area": 459, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "British Virgin Islands", + "pop2021": "30.4210", + "area": 151, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Gibraltar", + "pop2021": "33.6980", + "area": 6, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "San Marino", + "pop2021": "34.0170", + "area": 61, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Liechtenstein", + "pop2021": "38.2500", + "area": 160, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Turks and Caicos Islands", + "pop2021": "39.2310", + "area": 948, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Saint Martin", + "pop2021": "39.2340", + "area": 53, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Monaco", + "pop2021": "39.5110", + "area": 2, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Sint Maarten", + "pop2021": "43.4120", + "area": 34, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Faroe Islands", + "pop2021": "49.0490", + "area": 1393, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Saint Kitts and Nevis", + "pop2021": "53.5440", + "area": 261, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "American Samoa", + "pop2021": "55.1000", + "area": 199, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Greenland", + "pop2021": "56.8770", + "area": 2166086, + "region": "Americas", + "subregion": "Northern America" + }, + { + "country": "Northern Mariana Islands", + "pop2021": "57.9170", + "area": 464, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Marshall Islands", + "pop2021": "59.6100", + "area": 181, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Bermuda", + "pop2021": "62.0900", + "area": 54, + "region": "Americas", + "subregion": "Northern America" + }, + { + "country": "Cayman Islands", + "pop2021": "66.4970", + "area": 264, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Dominica", + "pop2021": "72.1670", + "area": 751, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Andorra", + "pop2021": "77.3550", + "area": 468, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Isle of Man", + "pop2021": "85.4100", + "area": 572, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Antigua and Barbuda", + "pop2021": "98.7310", + "area": 442, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Seychelles", + "pop2021": "98.9080", + "area": 452, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "United States Virgin Islands", + "pop2021": "104.2260", + "area": 347, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Tonga", + "pop2021": "106.7600", + "area": 747, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Aruba", + "pop2021": "107.2040", + "area": 180, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Saint Vincent and the Grenadines", + "pop2021": "111.2630", + "area": 389, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Grenada", + "pop2021": "113.0210", + "area": 344, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Micronesia", + "pop2021": "116.2540", + "area": 702, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Kiribati", + "pop2021": "121.3920", + "area": 811, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Curacao", + "pop2021": "164.7980", + "area": 444, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Guam", + "pop2021": "170.1790", + "area": 549, + "region": "Oceania", + "subregion": "Micronesia" + }, + { + "country": "Saint Lucia", + "pop2021": "184.4000", + "area": 616, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Samoa", + "pop2021": "200.1490", + "area": 2842, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Sao Tome and Principe", + "pop2021": "223.3680", + "area": 964, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Mayotte", + "pop2021": "279.5150", + "area": 374, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "French Polynesia", + "pop2021": "282.5300", + "area": 4167, + "region": "Oceania", + "subregion": "Polynesia" + }, + { + "country": "Barbados", + "pop2021": "287.7110", + "area": 430, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "New Caledonia", + "pop2021": "288.2180", + "area": 18575, + "region": "Oceania", + "subregion": "Melanesia" + }, + { + "country": "French Guiana", + "pop2021": "306.4480", + "area": 83534, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Vanuatu", + "pop2021": "314.4640", + "area": 12189, + "region": "Oceania", + "subregion": "Melanesia" + }, + { + "country": "Iceland", + "pop2021": "343.3530", + "area": 103000, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Martinique", + "pop2021": "374.7450", + "area": 1128, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Bahamas", + "pop2021": "396.9130", + "area": 13943, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Guadeloupe", + "pop2021": "400.0200", + "area": 1628, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Belize", + "pop2021": "404.9140", + "area": 22966, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Brunei", + "pop2021": "441.5320", + "area": 5765, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Malta", + "pop2021": "442.7840", + "area": 316, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Maldives", + "pop2021": "543.6170", + "area": 300, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Cape Verde", + "pop2021": "561.8980", + "area": 4033, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Suriname", + "pop2021": "591.8000", + "area": 163820, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Western Sahara", + "pop2021": "611.8750", + "area": 266000, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Montenegro", + "pop2021": "628.0530", + "area": 13812, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Luxembourg", + "pop2021": "634.8140", + "area": 2586, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Macau", + "pop2021": "658.3940", + "area": 30, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Solomon Islands", + "pop2021": "703.9960", + "area": 28896, + "region": "Oceania", + "subregion": "Melanesia" + }, + { + "country": "Bhutan", + "pop2021": "779.8980", + "area": 38394, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Guyana", + "pop2021": "790.3260", + "area": 214969, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Comoros", + "pop2021": "888.4510", + "area": 1862, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Reunion", + "pop2021": "901.6860", + "area": 2511, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Fiji", + "pop2021": "902.9060", + "area": 18272, + "region": "Oceania", + "subregion": "Melanesia" + }, + { + "country": "Djibouti", + "pop2021": "1002.1870", + "area": 23200, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Swaziland", + "pop2021": "1172.3620", + "area": 17364, + "region": "Africa", + "subregion": "Southern Africa" + }, + { + "country": "Cyprus", + "pop2021": "1215.5840", + "area": 9251, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Mauritius", + "pop2021": "1273.4330", + "area": 2040, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Estonia", + "pop2021": "1325.1850", + "area": 45227, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Timor-Leste", + "pop2021": "1343.8730", + "area": 14874, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Trinidad and Tobago", + "pop2021": "1403.3750", + "area": 5130, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Equatorial Guinea", + "pop2021": "1449.8960", + "area": 28051, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Bahrain", + "pop2021": "1748.2960", + "area": 765, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Latvia", + "pop2021": "1866.9420", + "area": 64559, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Guinea-Bissau", + "pop2021": "2015.4940", + "area": 36125, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Slovenia", + "pop2021": "2078.7240", + "area": 20273, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Macedonia", + "pop2021": "2082.6580", + "area": 25713, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Lesotho", + "pop2021": "2159.0790", + "area": 30355, + "region": "Africa", + "subregion": "Southern Africa" + }, + { + "country": "Gabon", + "pop2021": "2278.8250", + "area": 267668, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Botswana", + "pop2021": "2397.2410", + "area": 582000, + "region": "Africa", + "subregion": "Southern Africa" + }, + { + "country": "Gambia", + "pop2021": "2486.9450", + "area": 10689, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Namibia", + "pop2021": "2587.3440", + "area": 825615, + "region": "Africa", + "subregion": "Southern Africa" + }, + { + "country": "Lithuania", + "pop2021": "2689.8620", + "area": 65300, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Puerto Rico", + "pop2021": "2828.2550", + "area": 8870, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Albania", + "pop2021": "2872.9330", + "area": 28748, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Qatar", + "pop2021": "2930.5280", + "area": 11586, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Armenia", + "pop2021": "2968.1270", + "area": 29743, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Jamaica", + "pop2021": "2973.4630", + "area": 10991, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Bosnia and Herzegovina", + "pop2021": "3263.4660", + "area": 51209, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Mongolia", + "pop2021": "3329.2890", + "area": 1564110, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Uruguay", + "pop2021": "3485.1510", + "area": 181034, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Eritrea", + "pop2021": "3601.4670", + "area": 117600, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Georgia", + "pop2021": "3979.7650", + "area": 69700, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Moldova", + "pop2021": "4024.0190", + "area": 33846, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Croatia", + "pop2021": "4081.6510", + "area": 56594, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Kuwait", + "pop2021": "4328.5500", + "area": 17818, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Panama", + "pop2021": "4381.5790", + "area": 75417, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Mauritania", + "pop2021": "4775.1190", + "area": 1030700, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "New Zealand", + "pop2021": "4860.6430", + "area": 270467, + "region": "Oceania", + "subregion": "Australia and New Zealand" + }, + { + "country": "Central African Republic", + "pop2021": "4919.9810", + "area": 622984, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Ireland", + "pop2021": "4982.9070", + "area": 70273, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Costa Rica", + "pop2021": "5139.0520", + "area": 51100, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Liberia", + "pop2021": "5180.2030", + "area": 111369, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Palestine", + "pop2021": "5222.7480", + "area": 6220, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Oman", + "pop2021": "5223.3750", + "area": 309500, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Slovakia", + "pop2021": "5460.7210", + "area": 49037, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Norway", + "pop2021": "5465.6300", + "area": 323802, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Finland", + "pop2021": "5548.3600", + "area": 338424, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Republic of the Congo", + "pop2021": "5657.0130", + "area": 342000, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Denmark", + "pop2021": "5813.2980", + "area": 43094, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Singapore", + "pop2021": "5896.6860", + "area": 710, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Turkmenistan", + "pop2021": "6117.9240", + "area": 488100, + "region": "Asia", + "subregion": "Central Asia" + }, + { + "country": "El Salvador", + "pop2021": "6518.4990", + "area": 21041, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Kyrgyzstan", + "pop2021": "6628.3560", + "area": 199951, + "region": "Asia", + "subregion": "Central Asia" + }, + { + "country": "Nicaragua", + "pop2021": "6702.3850", + "area": 130373, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Lebanon", + "pop2021": "6769.1460", + "area": 10452, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Bulgaria", + "pop2021": "6896.6630", + "area": 110879, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Libya", + "pop2021": "6958.5320", + "area": 1759540, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Paraguay", + "pop2021": "7219.6380", + "area": 406752, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Laos", + "pop2021": "7379.3580", + "area": 236800, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Hong Kong", + "pop2021": "7552.8100", + "area": 1104, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Sierra Leone", + "pop2021": "8141.3430", + "area": 71740, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Togo", + "pop2021": "8478.2500", + "area": 56785, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Serbia", + "pop2021": "8697.5500", + "area": 88361, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Switzerland", + "pop2021": "8715.4940", + "area": 41284, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Israel", + "pop2021": "8789.7740", + "area": 20770, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Austria", + "pop2021": "9043.0700", + "area": 83871, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Papua New Guinea", + "pop2021": "9119.0100", + "area": 462840, + "region": "Oceania", + "subregion": "Melanesia" + }, + { + "country": "Belarus", + "pop2021": "9442.8620", + "area": 207600, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Hungary", + "pop2021": "9634.1640", + "area": 93028, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Tajikistan", + "pop2021": "9749.6270", + "area": 143100, + "region": "Asia", + "subregion": "Central Asia" + }, + { + "country": "United Arab Emirates", + "pop2021": "9991.0890", + "area": 83600, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Honduras", + "pop2021": "10062.9910", + "area": 112492, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Sweden", + "pop2021": "10160.1690", + "area": 450295, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Portugal", + "pop2021": "10167.9250", + "area": 92090, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Azerbaijan", + "pop2021": "10223.3420", + "area": 86600, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Jordan", + "pop2021": "10269.0210", + "area": 89342, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Greece", + "pop2021": "10370.7440", + "area": 131990, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Czech Republic", + "pop2021": "10724.5550", + "area": 78865, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Dominican Republic", + "pop2021": "10953.7030", + "area": 48671, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Cuba", + "pop2021": "11317.5050", + "area": 109884, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "South Sudan", + "pop2021": "11381.3780", + "area": 619745, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Haiti", + "pop2021": "11541.6850", + "area": 27750, + "region": "Americas", + "subregion": "Caribbean" + }, + { + "country": "Belgium", + "pop2021": "11632.3260", + "area": 30528, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Bolivia", + "pop2021": "11832.9400", + "area": 1098581, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Tunisia", + "pop2021": "11935.7660", + "area": 163610, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Burundi", + "pop2021": "12255.4330", + "area": 27834, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Benin", + "pop2021": "12451.0400", + "area": 112622, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Rwanda", + "pop2021": "13276.5130", + "area": 26338, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Guinea", + "pop2021": "13497.2440", + "area": 245857, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Zimbabwe", + "pop2021": "15092.1710", + "area": 390757, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Somalia", + "pop2021": "16359.5040", + "area": 637657, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Chad", + "pop2021": "16914.9850", + "area": 1284000, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Cambodia", + "pop2021": "16946.4380", + "area": 181035, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Netherlands", + "pop2021": "17173.0990", + "area": 41850, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Senegal", + "pop2021": "17196.3010", + "area": 196722, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Ecuador", + "pop2021": "17888.4750", + "area": 276841, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Guatemala", + "pop2021": "18249.8600", + "area": 108889, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Syria", + "pop2021": "18275.7020", + "area": 185180, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Zambia", + "pop2021": "18920.6510", + "area": 752612, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Kazakhstan", + "pop2021": "18994.9620", + "area": 2724900, + "region": "Asia", + "subregion": "Central Asia" + }, + { + "country": "Romania", + "pop2021": "19127.7740", + "area": 238391, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Chile", + "pop2021": "19212.3610", + "area": 756102, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Malawi", + "pop2021": "19647.6840", + "area": 118484, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Mali", + "pop2021": "20855.7350", + "area": 1240192, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Burkina Faso", + "pop2021": "21497.0960", + "area": 272967, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Sri Lanka", + "pop2021": "21497.3100", + "area": 65610, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Taiwan", + "pop2021": "23855.0100", + "area": 36193, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Niger", + "pop2021": "25130.8170", + "area": 1267000, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Australia", + "pop2021": "25788.2150", + "area": 7692024, + "region": "Oceania", + "subregion": "Australia and New Zealand" + }, + { + "country": "North Korea", + "pop2021": "25887.0410", + "area": 120538, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Ivory Coast", + "pop2021": "27053.6290", + "area": 322463, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Cameroon", + "pop2021": "27224.2650", + "area": 475442, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Madagascar", + "pop2021": "28427.3280", + "area": 587041, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Venezuela", + "pop2021": "28704.9540", + "area": 916445, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Nepal", + "pop2021": "29674.9200", + "area": 147181, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Yemen", + "pop2021": "30490.6400", + "area": 527968, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Ghana", + "pop2021": "31732.1290", + "area": 238533, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Mozambique", + "pop2021": "32163.0470", + "area": 801590, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Malaysia", + "pop2021": "32776.1940", + "area": 330803, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Peru", + "pop2021": "33359.4180", + "area": 1285216, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Angola", + "pop2021": "33933.6100", + "area": 1246700, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Uzbekistan", + "pop2021": "33935.7630", + "area": 447400, + "region": "Asia", + "subregion": "Central Asia" + }, + { + "country": "Saudi Arabia", + "pop2021": "35340.6830", + "area": 2149690, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Morocco", + "pop2021": "37344.7950", + "area": 446550, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Poland", + "pop2021": "37797.0050", + "area": 312679, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Canada", + "pop2021": "38067.9030", + "area": 9984670, + "region": "Americas", + "subregion": "Northern America" + }, + { + "country": "Afghanistan", + "pop2021": "39835.4280", + "area": 652230, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Iraq", + "pop2021": "41179.3500", + "area": 438317, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "Ukraine", + "pop2021": "43466.8190", + "area": 603500, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Algeria", + "pop2021": "44616.6240", + "area": 2381741, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Sudan", + "pop2021": "44909.3530", + "area": 1886068, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Argentina", + "pop2021": "45605.8260", + "area": 2780400, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Spain", + "pop2021": "46745.2160", + "area": 505992, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Uganda", + "pop2021": "47123.5310", + "area": 241550, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Colombia", + "pop2021": "51265.8440", + "area": 1141748, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "South Korea", + "pop2021": "51305.1860", + "area": 100210, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Myanmar", + "pop2021": "54806.0120", + "area": 676578, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Kenya", + "pop2021": "54985.6980", + "area": 580367, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "South Africa", + "pop2021": "60041.9940", + "area": 1221037, + "region": "Africa", + "subregion": "Southern Africa" + }, + { + "country": "Italy", + "pop2021": "60367.4770", + "area": 301336, + "region": "Europe", + "subregion": "Southern Europe" + }, + { + "country": "Tanzania", + "pop2021": "61498.4370", + "area": 945087, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "France", + "pop2021": "65426.1790", + "area": 551695, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "United Kingdom", + "pop2021": "68207.1160", + "area": 242900, + "region": "Europe", + "subregion": "Northern Europe" + }, + { + "country": "Thailand", + "pop2021": "69950.8500", + "area": 513120, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Germany", + "pop2021": "83900.4730", + "area": 357114, + "region": "Europe", + "subregion": "Western Europe" + }, + { + "country": "Iran", + "pop2021": "85028.7590", + "area": 1648195, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Turkey", + "pop2021": "85042.7380", + "area": 783562, + "region": "Asia", + "subregion": "Western Asia" + }, + { + "country": "DR Congo", + "pop2021": "92377.9930", + "area": 2344858, + "region": "Africa", + "subregion": "Middle Africa" + }, + { + "country": "Vietnam", + "pop2021": "98168.8330", + "area": 331212, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Egypt", + "pop2021": "104258.3270", + "area": 1002450, + "region": "Africa", + "subregion": "Northern Africa" + }, + { + "country": "Philippines", + "pop2021": "111046.9130", + "area": 342353, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "Ethiopia", + "pop2021": "117876.2270", + "area": 1104300, + "region": "Africa", + "subregion": "Eastern Africa" + }, + { + "country": "Japan", + "pop2021": "126050.8040", + "area": 377930, + "region": "Asia", + "subregion": "Eastern Asia" + }, + { + "country": "Mexico", + "pop2021": "130262.2160", + "area": 1964375, + "region": "Americas", + "subregion": "Central America" + }, + { + "country": "Russia", + "pop2021": "145912.0250", + "area": 17098242, + "region": "Europe", + "subregion": "Eastern Europe" + }, + { + "country": "Bangladesh", + "pop2021": "166303.4980", + "area": 147570, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Nigeria", + "pop2021": "211400.7080", + "area": 923768, + "region": "Africa", + "subregion": "Western Africa" + }, + { + "country": "Brazil", + "pop2021": "213993.4370", + "area": 8515767, + "region": "Americas", + "subregion": "South America" + }, + { + "country": "Pakistan", + "pop2021": "225199.9370", + "area": 881912, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "Indonesia", + "pop2021": "276361.7830", + "area": 1904569, + "region": "Asia", + "subregion": "South-Eastern Asia" + }, + { + "country": "United States", + "pop2021": "332915.0730", + "area": 9372610, + "region": "Americas", + "subregion": "Northern America" + }, + { + "country": "India", + "pop2021": "1393409.0380", + "area": 3287590, + "region": "Asia", + "subregion": "Southern Asia" + }, + { + "country": "China", + "pop2021": "1444216.1070", + "area": 9706961, + "region": "Asia", + "subregion": "Eastern Asia" + } + ] \ No newline at end of file diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py new file mode 100644 index 0000000..961f01d --- /dev/null +++ b/examples/example3/local_adapter.py @@ -0,0 +1,55 @@ +import json + +from slugify import slugify + +from diffsync import DiffSync +from models import Region, Country + +COUNTRIES_FILE = "countries.json" + + +class LocalAdapter(DiffSync): + """DiffSync Adapter to Load the list of regions and countries from a local JSON file.""" + + region = Region + country = Country + + # Since all countries are associated with a region, we don't need to list country here + # When doing a diff or a sync between 2 adapters, + # diffsync will recursively check all models defined at the top level and their children. + top_level = ["region"] + + # Human readable name of the Adapter, + # mainly used when doing a diff to indicate where each data is coming from + type = "Local" + + def load(self, filename=COUNTRIES_FILE): + """Load all regions and countries from a local JSON file.""" + + data_file = open(filename, "r") + countries = json.loads(data_file.read()) + + # Load all regions first + # A Region object will be create for each region and it will be store inside the object with self.add + # To create a Region we are using "self.region" instead of "Region" directly to allow someone to extend this adapter without redefining everything. + region_names = set([country.get("region") for country in countries]) + for region in region_names: + self.add(self.region(slug=slugify(region), name=region)) + + # Load all countries + # A Country object will be create for each country, it will be store inside the object with self.add + # and it will be linked to its parent with parent.add_child(item) + for country in countries: + + # Retrive the parent region object from the internal cache. + region = self.get(obj=self.region, identifier=slugify(country.get("region"))) + + name = country.get("country") + item = self.country( + slug=slugify(name), name=name, subregion=country.get("subregion", None), region=region.slug + ) + self.add(item) + + region.add_child(item) + + data_file.close() diff --git a/examples/example3/main.py b/examples/example3/main.py new file mode 100644 index 0000000..27d1945 --- /dev/null +++ b/examples/example3/main.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""Main executable for DiffSync "example2".""" +import sys +import argparse +import pprint + +from diffsync import Diff +from diffsync.logging import enable_console_logging + +from local_adapter import LocalAdapter +from nautobot_adapter import NautobotAdapter + + +def main(): + """Demonstrate DiffSync behavior using the example backends provided.""" + parser = argparse.ArgumentParser("example1") + parser.add_argument("--verbosity", "-v", default=0, action="count") + parser.add_argument("--diff", action="store_true") + parser.add_argument("--sync", action="store_true") + args = parser.parse_args() + enable_console_logging(verbosity=args.verbosity) + + if not args.sync and not args.diff: + sys.exit("please select --diff or --sync") + + print("Initializing and loading Local Data ...") + local = LocalAdapter() + local.load() + # print(local.str()) + + print("Initializing and loading Nautobot Data ...") + nautobot = NautobotAdapter() + nautobot.load() + # print(nautobot.str()) + + if args.diff: + print("Calculating the Diff between the local adapter and Nautobot ...") + diff = nautobot.diff_from(local) + print(diff.str()) + + elif args.sync: + print("Updating the list of countries in Nautobot ...") + nautobot.sync_from(local) + + +if __name__ == "__main__": + main() diff --git a/examples/example3/models.py b/examples/example3/models.py new file mode 100644 index 0000000..ff48b29 --- /dev/null +++ b/examples/example3/models.py @@ -0,0 +1,34 @@ +from typing import List, Optional +from diffsync import DiffSyncModel + + +class Region(DiffSyncModel): + """Example model of a geographic region.""" + + _modelname = "region" + _identifiers = ("slug",) + _attributes = ("name",) + + # By listing country as a child to Region + # DiffSync will be able to recursively compare all regions including all their children + _children = {"country": "countries"} + + slug: str + name: str + countries: List[str] = list() + + +class Country(DiffSyncModel): + """Example model of a Country. + + A must be part of a region and can be also associated with a subregion. + """ + + _modelname = "country" + _identifiers = ("slug",) + _attributes = ("name", "region", "subregion") + + slug: str + name: str + region: str + subregion: Optional[str] diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py new file mode 100644 index 0000000..a3a891f --- /dev/null +++ b/examples/example3/nautobot_adapter.py @@ -0,0 +1,167 @@ +import os +import pynautobot + +from diffsync import DiffSync +from models import Region, Country + +NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com") +NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + +class NautobotRegion(Region): + """Extend the Region object to store Nautobot specific information. + + Region are represented in Nautobot as a dcim.region object without parent. + """ + + remote_id: str + """Store the nautobot uuid in the object to allow update and delete of existing object.""" + + +class NautobotCountry(Country): + """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE. + + Country are represented in Nautobot as a dcim.region object as well but a country must have a parent. + Subregion information will be store in the description of the object in Nautobot + """ + + remote_id: str + """Store the nautobot uuid in the object to allow update and delete of existing object.""" + + @classmethod + def create(cls, diffsync: "DiffSync", ids: dict, attrs: dict): + """Create a country object in Nautobot. + + Args: + diffsync: The master data store for other DiffSyncModel instances that we might need to reference + ids: Dictionary of unique-identifiers needed to create the new object + attrs: Dictionary of additional attributes to set on the new object + + Returns: + NautobotCountry: DiffSync object newly created + """ + + # Retrieve the parent region in internal cache to access its UUID + # because the UUID is required to associate the object to its parent region in Nautobot + region = diffsync.get(diffsync.region, attrs.get("region")) + + # Create the new country in Nautobot and attach it to its parent + try: + country = diffsync.nautobot.dcim.regions.create( + slug=ids.get("slug"), + name=attrs.get("name"), + description=attrs.get("subregion", None), + parent=region.remote_id, + ) + print(f"Created country : {ids} | {attrs} | {country.id}") + + except pynautobot.core.query.RequestError as exc: + print(f"Unable to create country {ids} | {attrs} | {exc}") + return None + + # Add the newly created remote_id and create the internal object for this resource. + attrs["remote_id"] = country.id + item = super().create(ids=ids, diffsync=diffsync, attrs=attrs) + return item + + def update(self, attrs: dict): + """Update a country object in Nautobot. + + Args: + attrs: Dictionary of attributes to update on the object + + Returns: + DiffSyncModel: this instance, if all data was successfully updated. + None: if data updates failed in such a way that child objects of this model should not be modified. + + Raises: + ObjectNotUpdated: if an error occurred. + """ + + # Retrive the pynautobot object from Nautobot since we only have the UUID internally + remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) + + # Convert the internal attrs to Nautobot format + nautobot_attrs = {} + if "subregion" in attrs: + nautobot_attrs["description"] = attrs.get("subregion") + if "name" in attrs: + nautobot_attrs["name"] = attrs.get("name") + + if nautobot_attrs: + remote.update(data=nautobot_attrs) + print(f"Updated Country {self.slug} | {attrs}") + + return super().update(attrs) + + def delete(self): + """Delete a country object in Nautobot. + + Returns: + NautobotCountry: DiffSync object + """ + # Retrieve the pynautobot object and delete the object in Nautobot + remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) + remote.delete() + + super().delete() + return self + + +class NautobotAdapter(DiffSync): + """Example of a DiffSync adapter implementation.""" + + # We are using NautobotCountry and NautobotRegion instead of Region and Country + # because we are using these classes to manage the logic to integrate with Nautobot + # NautobotRegion is just a small extension to store the UUID and do not support any CRUD operation toward Nautobot + # NautobotCountry support the creation, update or deletion of a country in Nautobot + region = NautobotRegion + country = NautobotCountry + + # Since all countries are associated with a region, we don't need to list country here + # When doing a diff or a sync between 2 adapters, + # diffsync will recursively check all models defined at the top level and their children. + top_level = ["region"] + + # Human readable name of the Adapter, + # mainly used when doing a diff to indicate where each data is coming from + type = "Nautobot" + + def load(self): + """Load all data from Nautobot into the internal cache after transformation.""" + + # Initialize pynautobot to interact with Nautobot and store it within the adapter + # to reuse it later + self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN,) + + # Pull all regions from Nautobot, which includes all regions and all countries + regions = self.nautobot.dcim.regions.all() + + # Extract Region first (top level object without parent) + for region in regions: + if region.parent: + continue + + # We are excluding the networktocode because it's not present in the local file + if region.slug == "networktocode": + continue + + item = self.region(slug=region.slug, name=region.name, remote_id=region.id) + self.add(item) + + # Extract All countries (second level, country must have a parent) + for country in regions: + if not country.parent: + continue + + parent = self.get(self.region, country.parent.slug) + + item = self.country( + slug=country.slug, + name=country.name, + region=parent.slug, + subregion=country.description, + remote_id=country.id, + ) + self.add(item) + parent.add_child(item) diff --git a/examples/example3/requirements.txt b/examples/example3/requirements.txt new file mode 100644 index 0000000..5fb10fc --- /dev/null +++ b/examples/example3/requirements.txt @@ -0,0 +1,2 @@ +python-slugify +pynautobot From 9b0ffb1a0b5695f91e2952d16d869170d51803f2 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 8 Oct 2021 17:15:04 -0400 Subject: [PATCH 02/25] Replace sub region with population, add diff and flags --- examples/example3/README.md | 17 +++- examples/example3/diff.py | 15 +++ examples/example3/local_adapter.py | 8 +- examples/example3/main.py | 10 +- examples/example3/models.py | 6 +- examples/example3/nautobot_adapter.py | 130 +++++--------------------- examples/example3/nautobot_models.py | 104 +++++++++++++++++++++ 7 files changed, 170 insertions(+), 120 deletions(-) create mode 100644 examples/example3/diff.py create mode 100644 examples/example3/nautobot_models.py diff --git a/examples/example3/README.md b/examples/example3/README.md index bf668c6..a898ba2 100644 --- a/examples/example3/README.md +++ b/examples/example3/README.md @@ -4,7 +4,7 @@ This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like via a REST API like Nautobot. For this example, we have a shared model for Region and Country defined in `models.py`. -A Country must be associated with a Region and can be part of a Subregion too. +A country must be part of a region and has an attribute to capture its population. The comparison and synchronization of dataset is done between a local JSON file and the [public instance of Nautobot](https://demo.nautobot.com). @@ -16,17 +16,26 @@ to use this example you must have some dependencies installed, please make sure pip install -r requirements.txt ``` +## Setup the environment + +By default this example will interact with the public sandbox of Nautobot at https://demo.nautobot.com but you can use your own version of Nautobot by providing a new URL and a new API token using the environment variables `NAUTOBOT_URL` & `NAUTOBOT_TOKEN` + +``` +export NAUTOBOT_URL = "https://demo.nautobot.com" +export NAUTOBOT_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +``` + ## Try the example The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define. After the first sync, the diff should show no difference. -At this point, Diffsync will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode. +At this point, `Diffsync` will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode. ``` ### DIFF Compare the data between Nautobot and the local JSON file. -main.py --diff +python main.py --diff ### SYNC Update the list of country in Nautobot. -main.py --sync +python main.py --sync ``` diff --git a/examples/example3/diff.py b/examples/example3/diff.py new file mode 100644 index 0000000..a7b082e --- /dev/null +++ b/examples/example3/diff.py @@ -0,0 +1,15 @@ +from diffsync.diff import Diff + + +class AlphabeticalOrderDiff(Diff): + """Simple diff to return all children country in alphabetical order.""" + + @classmethod + def order_children_default(cls, children): + """Simple diff to return all children in alphabetical order.""" + for child_name, child in sorted(children.items()): + + # it's possible to access additional information about the object + # like child.action can be "update", "create" or "delete" + + yield children[child_name] diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index 961f01d..3467971 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -45,9 +45,11 @@ def load(self, filename=COUNTRIES_FILE): region = self.get(obj=self.region, identifier=slugify(country.get("region"))) name = country.get("country") - item = self.country( - slug=slugify(name), name=name, subregion=country.get("subregion", None), region=region.slug - ) + + # The population is store in thousands in the local file so we need to convert it + population = int(float(country.get("pop2021")) * 1000) + + item = self.country(slug=slugify(name), name=name, population=population, region=region.slug) self.add(item) region.add_child(item) diff --git a/examples/example3/main.py b/examples/example3/main.py index 27d1945..7225ead 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -5,10 +5,12 @@ import pprint from diffsync import Diff +from diffsync.enum import DiffSyncFlags from diffsync.logging import enable_console_logging from local_adapter import LocalAdapter from nautobot_adapter import NautobotAdapter +from diff import AlphabeticalOrderDiff def main(): @@ -26,21 +28,21 @@ def main(): print("Initializing and loading Local Data ...") local = LocalAdapter() local.load() - # print(local.str()) print("Initializing and loading Nautobot Data ...") nautobot = NautobotAdapter() nautobot.load() - # print(nautobot.str()) + + flags = DiffSyncFlags.SKIP_UNMATCHED_DST if args.diff: print("Calculating the Diff between the local adapter and Nautobot ...") - diff = nautobot.diff_from(local) + diff = nautobot.diff_from(local, flags=flags, diff_class=AlphabeticalOrderDiff) print(diff.str()) elif args.sync: print("Updating the list of countries in Nautobot ...") - nautobot.sync_from(local) + nautobot.sync_from(local, flags=flags, diff_class=AlphabeticalOrderDiff) if __name__ == "__main__": diff --git a/examples/example3/models.py b/examples/example3/models.py index ff48b29..7d779a4 100644 --- a/examples/example3/models.py +++ b/examples/example3/models.py @@ -21,14 +21,14 @@ class Region(DiffSyncModel): class Country(DiffSyncModel): """Example model of a Country. - A must be part of a region and can be also associated with a subregion. + A country must be part of a region and has an attribute to capture its population. """ _modelname = "country" _identifiers = ("slug",) - _attributes = ("name", "region", "subregion") + _attributes = ("name", "region", "population") slug: str name: str region: str - subregion: Optional[str] + population: Optional[int] diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index a3a891f..c52a97b 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -2,110 +2,21 @@ import pynautobot from diffsync import DiffSync -from models import Region, Country + +from nautobot_models import NautobotCountry, NautobotRegion NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com") NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - -class NautobotRegion(Region): - """Extend the Region object to store Nautobot specific information. - - Region are represented in Nautobot as a dcim.region object without parent. - """ - - remote_id: str - """Store the nautobot uuid in the object to allow update and delete of existing object.""" - - -class NautobotCountry(Country): - """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE. - - Country are represented in Nautobot as a dcim.region object as well but a country must have a parent. - Subregion information will be store in the description of the object in Nautobot - """ - - remote_id: str - """Store the nautobot uuid in the object to allow update and delete of existing object.""" - - @classmethod - def create(cls, diffsync: "DiffSync", ids: dict, attrs: dict): - """Create a country object in Nautobot. - - Args: - diffsync: The master data store for other DiffSyncModel instances that we might need to reference - ids: Dictionary of unique-identifiers needed to create the new object - attrs: Dictionary of additional attributes to set on the new object - - Returns: - NautobotCountry: DiffSync object newly created - """ - - # Retrieve the parent region in internal cache to access its UUID - # because the UUID is required to associate the object to its parent region in Nautobot - region = diffsync.get(diffsync.region, attrs.get("region")) - - # Create the new country in Nautobot and attach it to its parent - try: - country = diffsync.nautobot.dcim.regions.create( - slug=ids.get("slug"), - name=attrs.get("name"), - description=attrs.get("subregion", None), - parent=region.remote_id, - ) - print(f"Created country : {ids} | {attrs} | {country.id}") - - except pynautobot.core.query.RequestError as exc: - print(f"Unable to create country {ids} | {attrs} | {exc}") - return None - - # Add the newly created remote_id and create the internal object for this resource. - attrs["remote_id"] = country.id - item = super().create(ids=ids, diffsync=diffsync, attrs=attrs) - return item - - def update(self, attrs: dict): - """Update a country object in Nautobot. - - Args: - attrs: Dictionary of attributes to update on the object - - Returns: - DiffSyncModel: this instance, if all data was successfully updated. - None: if data updates failed in such a way that child objects of this model should not be modified. - - Raises: - ObjectNotUpdated: if an error occurred. - """ - - # Retrive the pynautobot object from Nautobot since we only have the UUID internally - remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) - - # Convert the internal attrs to Nautobot format - nautobot_attrs = {} - if "subregion" in attrs: - nautobot_attrs["description"] = attrs.get("subregion") - if "name" in attrs: - nautobot_attrs["name"] = attrs.get("name") - - if nautobot_attrs: - remote.update(data=nautobot_attrs) - print(f"Updated Country {self.slug} | {attrs}") - - return super().update(attrs) - - def delete(self): - """Delete a country object in Nautobot. - - Returns: - NautobotCountry: DiffSync object - """ - # Retrieve the pynautobot object and delete the object in Nautobot - remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) - remote.delete() - - super().delete() - return self +CUSTOM_FIELDS = [ + { + "name": "country_population", + "display": "Population (nbr people)", + "content_types": ["dcim.region"], + "type": "integer", + "description": "Number of inhabitant per country", + } +] class NautobotAdapter(DiffSync): @@ -132,7 +43,7 @@ def load(self): # Initialize pynautobot to interact with Nautobot and store it within the adapter # to reuse it later - self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN,) + self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN) # Pull all regions from Nautobot, which includes all regions and all countries regions = self.nautobot.dcim.regions.all() @@ -142,10 +53,6 @@ def load(self): if region.parent: continue - # We are excluding the networktocode because it's not present in the local file - if region.slug == "networktocode": - continue - item = self.region(slug=region.slug, name=region.name, remote_id=region.id) self.add(item) @@ -160,8 +67,19 @@ def load(self): slug=country.slug, name=country.name, region=parent.slug, - subregion=country.description, + population=country.custom_fields.get("country_population", None), remote_id=country.id, ) self.add(item) parent.add_child(item) + + def sync_from(self, *args, **kwargs): + """Sync the data with Nautobot but first ensure that all the required Custom fields are present in Nautobot.""" + + # Check if all required custom fields exist, create them if they don't + for custom_field in CUSTOM_FIELDS: + nb_cfs = self.cfs = self.nautobot.extras.custom_fields.filter(name=custom_field.get("name")) + if not nb_cfs: + self.nautobot.extras.custom_fields.create(**custom_field) + + super().sync_from(*args, **kwargs) diff --git a/examples/example3/nautobot_models.py b/examples/example3/nautobot_models.py new file mode 100644 index 0000000..22af7ba --- /dev/null +++ b/examples/example3/nautobot_models.py @@ -0,0 +1,104 @@ +import os +import pynautobot + + +from diffsync import DiffSync +from models import Region, Country + + +class NautobotRegion(Region): + """Extend the Region object to store Nautobot specific information. + + Region are represented in Nautobot as a dcim.region object without parent. + """ + + remote_id: str + """Store the nautobot uuid in the object to allow update and delete of existing object.""" + + +class NautobotCountry(Country): + """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE. + + Country are represented in Nautobot as a dcim.region object as well but a country must have a parent. + Subregion information will be store in the description of the object in Nautobot + """ + + remote_id: str + """Store the nautobot uuid in the object to allow update and delete of existing object.""" + + @classmethod + def create(cls, diffsync: DiffSync, ids: dict, attrs: dict): + """Create a country object in Nautobot. + + Args: + diffsync: The master data store for other DiffSyncModel instances that we might need to reference + ids: Dictionary of unique-identifiers needed to create the new object + attrs: Dictionary of additional attributes to set on the new object + + Returns: + NautobotCountry: DiffSync object newly created + """ + + # Retrieve the parent region in internal cache to access its UUID + # because the UUID is required to associate the object to its parent region in Nautobot + region = diffsync.get(diffsync.region, attrs.get("region")) + + # Create the new country in Nautobot and attach it to its parent + try: + country = diffsync.nautobot.dcim.regions.create( + slug=ids.get("slug"), + name=attrs.get("name"), + custom_fields=dict(population=attrs.get("population")), + parent=region.remote_id, + ) + print(f"Created country : {ids} | {attrs} | {country.id}") + + except pynautobot.core.query.RequestError as exc: + print(f"Unable to create country {ids} | {attrs} | {exc}") + return None + + # Add the newly created remote_id and create the internal object for this resource. + attrs["remote_id"] = country.id + item = super().create(ids=ids, diffsync=diffsync, attrs=attrs) + return item + + def update(self, attrs: dict): + """Update a country object in Nautobot. + + Args: + attrs: Dictionary of attributes to update on the object + + Returns: + DiffSyncModel: this instance, if all data was successfully updated. + None: if data updates failed in such a way that child objects of this model should not be modified. + + Raises: + ObjectNotUpdated: if an error occurred. + """ + + # Retrive the pynautobot object from Nautobot since we only have the UUID internally + remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) + + # Convert the internal attrs to Nautobot format + if "population" in attrs: + remote.custom_fields["country_population"] = attrs.get("population") + if "name" in attrs: + remote.name = attrs.get("name") + + remote.save() + print(f"Updated Country {self.slug} | {attrs}") + + return super().update(attrs) + + def delete(self): + """Delete a country object in Nautobot. + + Returns: + NautobotCountry: DiffSync object + """ + # Retrieve the pynautobot object and delete the object in Nautobot + remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) + remote.delete() + + super().delete() + return self From 5288b5fcadc29f041f73a7d7cc28fbb06bdaf768 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 11 Oct 2021 09:01:24 -0400 Subject: [PATCH 03/25] Pylint and pydocstyle --- examples/example3/diff.py | 1 + examples/example3/local_adapter.py | 7 ++++--- examples/example3/main.py | 2 -- examples/example3/models.py | 1 + examples/example3/nautobot_adapter.py | 12 +++++++----- examples/example3/nautobot_models.py | 7 ++----- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/example3/diff.py b/examples/example3/diff.py index a7b082e..13f1c62 100644 --- a/examples/example3/diff.py +++ b/examples/example3/diff.py @@ -1,3 +1,4 @@ +"""Custom Diff class for DiffSync to influence the behavior of the core Engine.""" from diffsync.diff import Diff diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index 3467971..e8fbc35 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -1,9 +1,11 @@ +"""DiffSync adapter to load data from a local file.""" import json from slugify import slugify -from diffsync import DiffSync from models import Region, Country +from diffsync import DiffSync + COUNTRIES_FILE = "countries.json" @@ -25,14 +27,13 @@ class LocalAdapter(DiffSync): def load(self, filename=COUNTRIES_FILE): """Load all regions and countries from a local JSON file.""" - data_file = open(filename, "r") countries = json.loads(data_file.read()) # Load all regions first # A Region object will be create for each region and it will be store inside the object with self.add # To create a Region we are using "self.region" instead of "Region" directly to allow someone to extend this adapter without redefining everything. - region_names = set([country.get("region") for country in countries]) + region_names = {country.get("region") for country in countries} for region in region_names: self.add(self.region(slug=slugify(region), name=region)) diff --git a/examples/example3/main.py b/examples/example3/main.py index 7225ead..77ea97e 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -2,9 +2,7 @@ """Main executable for DiffSync "example2".""" import sys import argparse -import pprint -from diffsync import Diff from diffsync.enum import DiffSyncFlags from diffsync.logging import enable_console_logging diff --git a/examples/example3/models.py b/examples/example3/models.py index 7d779a4..baa03d0 100644 --- a/examples/example3/models.py +++ b/examples/example3/models.py @@ -1,3 +1,4 @@ +"""Main DiffSync models for example3.""" from typing import List, Optional from diffsync import DiffSyncModel diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index c52a97b..affb877 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -1,9 +1,11 @@ +"""DiffSync Adapter for Nautobot to manage regions.""" import os import pynautobot +from nautobot_models import NautobotCountry, NautobotRegion + from diffsync import DiffSync -from nautobot_models import NautobotCountry, NautobotRegion NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com") NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") @@ -40,10 +42,11 @@ class NautobotAdapter(DiffSync): def load(self): """Load all data from Nautobot into the internal cache after transformation.""" - # Initialize pynautobot to interact with Nautobot and store it within the adapter # to reuse it later - self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN) + self.nautobot = pynautobot.api( + url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN + ) # pylint: disable=attribute-defined-outside-init # Pull all regions from Nautobot, which includes all regions and all countries regions = self.nautobot.dcim.regions.all() @@ -75,10 +78,9 @@ def load(self): def sync_from(self, *args, **kwargs): """Sync the data with Nautobot but first ensure that all the required Custom fields are present in Nautobot.""" - # Check if all required custom fields exist, create them if they don't for custom_field in CUSTOM_FIELDS: - nb_cfs = self.cfs = self.nautobot.extras.custom_fields.filter(name=custom_field.get("name")) + nb_cfs = self.nautobot.extras.custom_fields.filter(name=custom_field.get("name")) if not nb_cfs: self.nautobot.extras.custom_fields.create(**custom_field) diff --git a/examples/example3/nautobot_models.py b/examples/example3/nautobot_models.py index 22af7ba..2a316bb 100644 --- a/examples/example3/nautobot_models.py +++ b/examples/example3/nautobot_models.py @@ -1,7 +1,6 @@ -import os +"""Extension of the Base model for the Nautobot DiffSync Adapter to manage the CRUD operations.""" import pynautobot - from diffsync import DiffSync from models import Region, Country @@ -18,7 +17,7 @@ class NautobotRegion(Region): class NautobotCountry(Country): """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE. - + Country are represented in Nautobot as a dcim.region object as well but a country must have a parent. Subregion information will be store in the description of the object in Nautobot """ @@ -38,7 +37,6 @@ def create(cls, diffsync: DiffSync, ids: dict, attrs: dict): Returns: NautobotCountry: DiffSync object newly created """ - # Retrieve the parent region in internal cache to access its UUID # because the UUID is required to associate the object to its parent region in Nautobot region = diffsync.get(diffsync.region, attrs.get("region")) @@ -75,7 +73,6 @@ def update(self, attrs: dict): Raises: ObjectNotUpdated: if an error occurred. """ - # Retrive the pynautobot object from Nautobot since we only have the UUID internally remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id) From 144820d70e641c351575653e504d3f3eefd93d76 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 11 Oct 2021 09:13:28 -0400 Subject: [PATCH 04/25] exclude examples and docs from mypy --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index e10027b..15e152d 100644 --- a/tasks.py +++ b/tasks.py @@ -197,7 +197,7 @@ def mypy(context, name=NAME, image_ver=IMAGE_VER, local=INVOKE_LOCAL): """ # pty is set to true to properly run the docker commands due to the invocation process of docker # https://docs.pyinvoke.org/en/latest/api/runners.html - Search for pty for more information - exec_cmd = 'find . -name "*.py" | xargs mypy --show-error-codes' + exec_cmd = 'find . -name "*.py" -not -path "*/examples/*" -not -path "*/docs/*" | xargs mypy --show-error-codes' run_cmd(context, exec_cmd, name, image_ver, local) From 8c2d17ef7eeb9d85a8340e0fc0e0e75665361964 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Mon, 11 Oct 2021 14:21:31 -0400 Subject: [PATCH 05/25] exclude examples and docs from pylint --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 15e152d..19ca63c 100644 --- a/tasks.py +++ b/tasks.py @@ -213,7 +213,7 @@ def pylint(context, name=NAME, image_ver=IMAGE_VER, local=INVOKE_LOCAL): """ # pty is set to true to properly run the docker commands due to the invocation process of docker # https://docs.pyinvoke.org/en/latest/api/runners.html - Search for pty for more information - exec_cmd = 'find . -name "*.py" | xargs pylint' + exec_cmd = 'find . -name "*.py" -not -path "*/examples/*" -not -path "*/docs/*" | xargs pylint' run_cmd(context, exec_cmd, name, image_ver, local) From 8a4cbd9c8b4cbffc728a000cb5257c16eeaf7b79 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 10:19:51 -0400 Subject: [PATCH 06/25] Update examples/example3/main.py Co-authored-by: Glenn Matthews --- examples/example3/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/example3/main.py b/examples/example3/main.py index 77ea97e..08cc8ca 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -31,6 +31,7 @@ def main(): nautobot = NautobotAdapter() nautobot.load() + # If a Region exists in Nautobot (the "destination") but not in the local data, skip it, rather than deleting it flags = DiffSyncFlags.SKIP_UNMATCHED_DST if args.diff: From 23ce5b3a18f88286f7633724900e50465a86fdc9 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 15:31:24 -0400 Subject: [PATCH 07/25] Update examples/example3/diff.py Co-authored-by: Glenn Matthews --- examples/example3/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/diff.py b/examples/example3/diff.py index 13f1c62..fd9f82c 100644 --- a/examples/example3/diff.py +++ b/examples/example3/diff.py @@ -13,4 +13,4 @@ def order_children_default(cls, children): # it's possible to access additional information about the object # like child.action can be "update", "create" or "delete" - yield children[child_name] + yield child From 75781960b08280c0539b926ce9929f26e57ffd81 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:12:22 -0400 Subject: [PATCH 08/25] Update examples/example3/local_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/local_adapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index e8fbc35..0ae2b91 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -13,6 +13,8 @@ class LocalAdapter(DiffSync): """DiffSync Adapter to Load the list of regions and countries from a local JSON file.""" + # Define all data models that this adapter makes use of. + # Note that the variable names ("region", "country") need to match between DiffSync Adapter classes. region = Region country = Country From 0685de8bc0711dab6ab6eaa38829ba9e1cfef7ba Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:12:51 -0400 Subject: [PATCH 09/25] Update examples/example3/local_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/local_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index 0ae2b91..4bc266f 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -29,8 +29,8 @@ class LocalAdapter(DiffSync): def load(self, filename=COUNTRIES_FILE): """Load all regions and countries from a local JSON file.""" - data_file = open(filename, "r") - countries = json.loads(data_file.read()) + with open(filename, "r") as data_file: + countries = json.load(data_file) # Load all regions first # A Region object will be create for each region and it will be store inside the object with self.add From f91d5c4e591e745ea8160588bb97ae7d31aaa16a Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:13:08 -0400 Subject: [PATCH 10/25] Update examples/example3/local_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/local_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index 4bc266f..bd85f4b 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -33,7 +33,7 @@ def load(self, filename=COUNTRIES_FILE): countries = json.load(data_file) # Load all regions first - # A Region object will be create for each region and it will be store inside the object with self.add + # A Region object will be created for each region and it will be stored inside the adapter with self.add() # To create a Region we are using "self.region" instead of "Region" directly to allow someone to extend this adapter without redefining everything. region_names = {country.get("region") for country in countries} for region in region_names: From db21ce17801815d002a02432f0b3eda0795c71a6 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:13:17 -0400 Subject: [PATCH 11/25] Update examples/example3/README.md Co-authored-by: Glenn Matthews --- examples/example3/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/README.md b/examples/example3/README.md index a898ba2..95fdf5b 100644 --- a/examples/example3/README.md +++ b/examples/example3/README.md @@ -28,7 +28,7 @@ export NAUTOBOT_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ## Try the example The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define. -After the first sync, the diff should show no difference. +After the first sync, on subsequent runs, the diff should show no changes. At this point, `Diffsync` will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode. ``` From 1eb9b3e901a55ce07c2b3d01fda87dd957f8ee36 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:13:31 -0400 Subject: [PATCH 12/25] Update examples/example3/README.md Co-authored-by: Glenn Matthews --- examples/example3/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/README.md b/examples/example3/README.md index 95fdf5b..c9d5aca 100644 --- a/examples/example3/README.md +++ b/examples/example3/README.md @@ -27,7 +27,7 @@ export NAUTOBOT_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ## Try the example -The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define. +The first time you run this example, a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion defined. After the first sync, on subsequent runs, the diff should show no changes. At this point, `Diffsync` will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode. From 29a09955423831da40d11ef1a5487e1c7f2b008c Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:13:44 -0400 Subject: [PATCH 13/25] Update examples/example3/README.md Co-authored-by: Glenn Matthews --- examples/example3/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/README.md b/examples/example3/README.md index c9d5aca..d6d55fb 100644 --- a/examples/example3/README.md +++ b/examples/example3/README.md @@ -1,7 +1,7 @@ # Example 3 -This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like via a REST API like Nautobot. +This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like [Nautobot](https://nautobot.readthedocs.io) via a REST API. For this example, we have a shared model for Region and Country defined in `models.py`. A country must be part of a region and has an attribute to capture its population. From 9924b1da6771e6baa18e49e22c4a60a4dbdb4863 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:14:03 -0400 Subject: [PATCH 14/25] Update examples/example3/local_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/local_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index bd85f4b..c6ecc73 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -40,7 +40,7 @@ def load(self, filename=COUNTRIES_FILE): self.add(self.region(slug=slugify(region), name=region)) # Load all countries - # A Country object will be create for each country, it will be store inside the object with self.add + # A Country object will be created for each country, it will be stored inside the adapter with self.add(), # and it will be linked to its parent with parent.add_child(item) for country in countries: From f1382c20deee73a6dbfe7a6fc3245abe10190522 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:14:19 -0400 Subject: [PATCH 15/25] Update examples/example3/local_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/local_adapter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index c6ecc73..cd5ee7a 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -57,4 +57,3 @@ def load(self, filename=COUNTRIES_FILE): region.add_child(item) - data_file.close() From 1f73b747b673842678358cad8af1342c822b9cad Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:15:02 -0400 Subject: [PATCH 16/25] Update examples/example3/main.py Co-authored-by: Glenn Matthews --- examples/example3/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/main.py b/examples/example3/main.py index 08cc8ca..40278df 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Main executable for DiffSync "example2".""" +"""Main executable for DiffSync "example3".""" import sys import argparse From bd43009ef0aa145f4376d01e90e65a4362da6017 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:15:24 -0400 Subject: [PATCH 17/25] Update examples/example3/main.py Co-authored-by: Glenn Matthews --- examples/example3/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/main.py b/examples/example3/main.py index 40278df..ecfd5ba 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -13,7 +13,7 @@ def main(): """Demonstrate DiffSync behavior using the example backends provided.""" - parser = argparse.ArgumentParser("example1") + parser = argparse.ArgumentParser("example3") parser.add_argument("--verbosity", "-v", default=0, action="count") parser.add_argument("--diff", action="store_true") parser.add_argument("--sync", action="store_true") From 38e8dd948d64d732db3cbfc4bcbc4f9af30c2049 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:15:47 -0400 Subject: [PATCH 18/25] Update examples/example3/main.py Co-authored-by: Glenn Matthews --- examples/example3/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/main.py b/examples/example3/main.py index ecfd5ba..0d74979 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -39,7 +39,7 @@ def main(): diff = nautobot.diff_from(local, flags=flags, diff_class=AlphabeticalOrderDiff) print(diff.str()) - elif args.sync: + if args.sync: print("Updating the list of countries in Nautobot ...") nautobot.sync_from(local, flags=flags, diff_class=AlphabeticalOrderDiff) From b0e8b5476b2a95d490da6b9c85a9e16b161c78af Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:16:18 -0400 Subject: [PATCH 19/25] Update examples/example3/nautobot_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/nautobot_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index affb877..9e45587 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -22,7 +22,7 @@ class NautobotAdapter(DiffSync): - """Example of a DiffSync adapter implementation.""" + """Example of a DiffSync adapter implementation using pynautobot to communicate with a remote Nautobot system.""" # We are using NautobotCountry and NautobotRegion instead of Region and Country # because we are using these classes to manage the logic to integrate with Nautobot From b99649dda2aaa340740558d42329851d53cb0a7c Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:16:30 -0400 Subject: [PATCH 20/25] Update examples/example3/nautobot_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/nautobot_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index 9e45587..2cdb38f 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -26,7 +26,7 @@ class NautobotAdapter(DiffSync): # We are using NautobotCountry and NautobotRegion instead of Region and Country # because we are using these classes to manage the logic to integrate with Nautobot - # NautobotRegion is just a small extension to store the UUID and do not support any CRUD operation toward Nautobot + # NautobotRegion is just a small extension to store the UUID and does not support any CRUD operation toward Nautobot # NautobotCountry support the creation, update or deletion of a country in Nautobot region = NautobotRegion country = NautobotCountry From 06ec2d20c38fb4b9b2d123960f4a9f76271c8082 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:16:52 -0400 Subject: [PATCH 21/25] Update examples/example3/nautobot_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/nautobot_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index 2cdb38f..e3f693f 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -27,7 +27,7 @@ class NautobotAdapter(DiffSync): # We are using NautobotCountry and NautobotRegion instead of Region and Country # because we are using these classes to manage the logic to integrate with Nautobot # NautobotRegion is just a small extension to store the UUID and does not support any CRUD operation toward Nautobot - # NautobotCountry support the creation, update or deletion of a country in Nautobot + # NautobotCountry supports the creation, update or deletion of a country in Nautobot region = NautobotRegion country = NautobotCountry From b3071763075d76866514cdc9b5f36235d996cbdd Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 12 Oct 2021 16:17:26 -0400 Subject: [PATCH 22/25] Update examples/example3/nautobot_adapter.py Co-authored-by: Glenn Matthews --- examples/example3/nautobot_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index e3f693f..8cb9e2c 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -31,9 +31,9 @@ class NautobotAdapter(DiffSync): region = NautobotRegion country = NautobotCountry - # Since all countries are associated with a region, we don't need to list country here # When doing a diff or a sync between 2 adapters, # diffsync will recursively check all models defined at the top level and their children. + # Since countries are defined as children of a region, we don't need to list country here top_level = ["region"] # Human readable name of the Adapter, From c91eeb54abb50b2ddc78cb73ba06f975da4b56db Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Thu, 14 Oct 2021 10:46:07 -0400 Subject: [PATCH 23/25] Update format with black --- examples/example3/local_adapter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index cd5ee7a..81d7f04 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -56,4 +56,3 @@ def load(self, filename=COUNTRIES_FILE): self.add(item) region.add_child(item) - From efd36a773ea71916553a6d5b8a4a3a32f2c52f92 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Thu, 14 Oct 2021 13:19:05 -0400 Subject: [PATCH 24/25] Reenable pylint on the examples folder --- examples/example1/backend_a.py | 2 +- examples/example1/backend_b.py | 2 +- examples/example1/backend_c.py | 2 +- examples/example3/diff.py | 2 +- examples/example3/local_adapter.py | 2 +- examples/example3/main.py | 6 +++--- examples/example3/nautobot_adapter.py | 7 +++---- examples/example3/nautobot_models.py | 6 +++++- tasks.py | 2 +- 9 files changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/example1/backend_a.py b/examples/example1/backend_a.py index bbe6af7..a92435b 100644 --- a/examples/example1/backend_a.py +++ b/examples/example1/backend_a.py @@ -17,7 +17,7 @@ # pylint: disable=wrong-import-order from diffsync import DiffSync -from models import Site, Device, Interface +from models import Site, Device, Interface # pylint: disable=no-name-in-module DATA = { "nyc": { diff --git a/examples/example1/backend_b.py b/examples/example1/backend_b.py index 4a0c7bd..6fc7c20 100644 --- a/examples/example1/backend_b.py +++ b/examples/example1/backend_b.py @@ -17,7 +17,7 @@ # pylint: disable=wrong-import-order from diffsync import DiffSync -from models import Site, Device, Interface +from models import Site, Device, Interface # pylint: disable=no-name-in-module DATA = { "atl": { diff --git a/examples/example1/backend_c.py b/examples/example1/backend_c.py index 1e59f8e..a964e96 100644 --- a/examples/example1/backend_c.py +++ b/examples/example1/backend_c.py @@ -17,7 +17,7 @@ # pylint: disable=wrong-import-order from diffsync import DiffSync -from models import Site, Device, Interface +from models import Site, Device, Interface # pylint: disable=no-name-in-module DATA = { "nyc": { diff --git a/examples/example3/diff.py b/examples/example3/diff.py index fd9f82c..6c87191 100644 --- a/examples/example3/diff.py +++ b/examples/example3/diff.py @@ -8,7 +8,7 @@ class AlphabeticalOrderDiff(Diff): @classmethod def order_children_default(cls, children): """Simple diff to return all children in alphabetical order.""" - for child_name, child in sorted(children.items()): + for child in sorted(children.values()): # it's possible to access additional information about the object # like child.action can be "update", "create" or "delete" diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index 81d7f04..c5bbb74 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -27,7 +27,7 @@ class LocalAdapter(DiffSync): # mainly used when doing a diff to indicate where each data is coming from type = "Local" - def load(self, filename=COUNTRIES_FILE): + def load(self, filename=COUNTRIES_FILE): # pylint: disable=arguments-differ """Load all regions and countries from a local JSON file.""" with open(filename, "r") as data_file: countries = json.load(data_file) diff --git a/examples/example3/main.py b/examples/example3/main.py index 0d74979..66dc6ff 100644 --- a/examples/example3/main.py +++ b/examples/example3/main.py @@ -3,13 +3,13 @@ import sys import argparse -from diffsync.enum import DiffSyncFlags -from diffsync.logging import enable_console_logging - from local_adapter import LocalAdapter from nautobot_adapter import NautobotAdapter from diff import AlphabeticalOrderDiff +from diffsync.enum import DiffSyncFlags +from diffsync.logging import enable_console_logging + def main(): """Demonstrate DiffSync behavior using the example backends provided.""" diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index 8cb9e2c..35831ba 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -6,6 +6,7 @@ from diffsync import DiffSync +# pylint: disable=attribute-defined-outside-init NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com") NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") @@ -44,9 +45,7 @@ def load(self): """Load all data from Nautobot into the internal cache after transformation.""" # Initialize pynautobot to interact with Nautobot and store it within the adapter # to reuse it later - self.nautobot = pynautobot.api( - url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN - ) # pylint: disable=attribute-defined-outside-init + self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN) # Pull all regions from Nautobot, which includes all regions and all countries regions = self.nautobot.dcim.regions.all() @@ -76,7 +75,7 @@ def load(self): self.add(item) parent.add_child(item) - def sync_from(self, *args, **kwargs): + def sync_from(self, *args, **kwargs): # pylint: disable=signature-differs """Sync the data with Nautobot but first ensure that all the required Custom fields are present in Nautobot.""" # Check if all required custom fields exist, create them if they don't for custom_field in CUSTOM_FIELDS: diff --git a/examples/example3/nautobot_models.py b/examples/example3/nautobot_models.py index 2a316bb..f397c86 100644 --- a/examples/example3/nautobot_models.py +++ b/examples/example3/nautobot_models.py @@ -1,9 +1,13 @@ """Extension of the Base model for the Nautobot DiffSync Adapter to manage the CRUD operations.""" import pynautobot -from diffsync import DiffSync from models import Region, Country +from diffsync import DiffSync + + +# pylint: disable=no-member + class NautobotRegion(Region): """Extend the Region object to store Nautobot specific information. diff --git a/tasks.py b/tasks.py index 19ca63c..15e152d 100644 --- a/tasks.py +++ b/tasks.py @@ -213,7 +213,7 @@ def pylint(context, name=NAME, image_ver=IMAGE_VER, local=INVOKE_LOCAL): """ # pty is set to true to properly run the docker commands due to the invocation process of docker # https://docs.pyinvoke.org/en/latest/api/runners.html - Search for pty for more information - exec_cmd = 'find . -name "*.py" -not -path "*/examples/*" -not -path "*/docs/*" | xargs pylint' + exec_cmd = 'find . -name "*.py" | xargs pylint' run_cmd(context, exec_cmd, name, image_ver, local) From e47dd691d9420f6e8b4ae8ecf1ff4fa45dac710c Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Thu, 14 Oct 2021 13:40:56 -0400 Subject: [PATCH 25/25] fix import error for pylint in example3 --- examples/example3/local_adapter.py | 2 +- examples/example3/nautobot_adapter.py | 2 +- examples/example3/nautobot_models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/example3/local_adapter.py b/examples/example3/local_adapter.py index c5bbb74..301f8db 100644 --- a/examples/example3/local_adapter.py +++ b/examples/example3/local_adapter.py @@ -1,7 +1,7 @@ """DiffSync adapter to load data from a local file.""" import json -from slugify import slugify +from slugify import slugify # pylint: disable=import-error from models import Region, Country from diffsync import DiffSync diff --git a/examples/example3/nautobot_adapter.py b/examples/example3/nautobot_adapter.py index 35831ba..e0fb386 100644 --- a/examples/example3/nautobot_adapter.py +++ b/examples/example3/nautobot_adapter.py @@ -1,6 +1,6 @@ """DiffSync Adapter for Nautobot to manage regions.""" import os -import pynautobot +import pynautobot # pylint: disable=import-error from nautobot_models import NautobotCountry, NautobotRegion diff --git a/examples/example3/nautobot_models.py b/examples/example3/nautobot_models.py index f397c86..face8b5 100644 --- a/examples/example3/nautobot_models.py +++ b/examples/example3/nautobot_models.py @@ -1,5 +1,5 @@ """Extension of the Base model for the Nautobot DiffSync Adapter to manage the CRUD operations.""" -import pynautobot +import pynautobot # pylint: disable=import-error from models import Region, Country