Network enhancements

gml_application_schema_toolbox/core/gmlas_postgis_db.py

@@ -7,7 +7,7 @@
  from gml_application_schema_toolbox.extlibs.qgis_processing_postgis import DbError
 
 # project package
-from gml_application_schema_toolbox.core.log_handler import PluginLogHandler
+from gml_application_schema_toolbox.toolbelt.log_handler import PlgLogger
 from gml_application_schema_toolbox.extlibs.qgis_processing_postgis import GeoDB
 
 # ############################################################################
@@ -34,7 +34,7 @@ def __str__(self):
 class GmlasPostgisDB(GeoDB):
  def __init__(self):
  # map to the plugin log handler
- self.plg_logger = PluginLogHandler()
+ self.plg_logger = PlgLogger()
 
  def _add_foreign_key_constraint(self, schema, foreign_key):
  if self._constraint_exists(

gml_application_schema_toolbox/core/load_gml_as_xml.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Library General Public
 # License along with this library; if not, see <http://www.gnu.org/licenses/>.
 
-
+# standard library
 import copy
 import os
 import re
@@ -42,8 +42,8 @@
 
 
 def load_as_xml_layer(
- xml_uri,
- is_remote,
+ xml_uri: str,
+ is_remote: bool,
  attributes={},
  geometry_mapping=None,
  output_local_file=None,
@@ -360,12 +360,12 @@ def is_layer_complex(layer):
 
  def load_complex_gml(
  self,
- xml_uri,
- is_remote,
+ xml_uri: str,
+ is_remote: bool,
  attributes={},
  geometry_mapping=None,
  logger=None,
- swap_xy=False,
+ swap_xy: bool = False,
  ):
  """
  :param xml_uri: the XML URI

gml_application_schema_toolbox/core/qgis_urlopener.py

@@ -1,69 +1,72 @@
-# Copyright (C) 2016 BRGM (http:///brgm.fr)
-# Copyright (C) 2016 Oslandia <infos@oslandia.com>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Library General Public License for more details.
-# You should have received a copy of the GNU Library General Public
-# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#! python3 # noqa: E265
 
-from io import BytesIO
+# ############################################################################
+# ########## Imports ###############
+# ##################################
 
-from qgis.core import QgsNetworkAccessManager
-from qgis.PyQt.QtCore import QEventLoop, QUrl
-from qgis.PyQt.QtNetwork import QNetworkAccessManager, QNetworkRequest
+# Standard library
+import logging
+from io import BytesIO
+from typing import Dict
 
+# project
 from gml_application_schema_toolbox.__about__ import __title__
 from gml_application_schema_toolbox.core.settings import settings
+from gml_application_schema_toolbox.toolbelt.log_handler import PlgLogger
+from gml_application_schema_toolbox.toolbelt.network_manager import (
+ NetworkAccessManager,
+ RequestsException,
+)
+
+# ############################################################################
+# ########## Globals ###############
+# ##################################
 
 __network_manager = None
+logger = logging.getLogger(__name__)
+plg_logger = PlgLogger()
+
 
+# ############################################################################
+# ########## Functions #############
+# ##################################
 
-def _sync_get(url):
- global __network_manager
- if __network_manager is None:
- __network_manager = QNetworkAccessManager()
- __network_manager.setProxy(QgsNetworkAccessManager.instance().proxy())
- pause = QEventLoop()
- req = QNetworkRequest(url)
- req.setRawHeader(b"Accept", b"application/xml")
- req.setRawHeader(
- b"Accept-Language", bytes(settings.value("default_language", "fr"), "utf8")
- )
- req.setRawHeader(
- b"User-Agent", bytes(settings.value("http_user_agent", __title__), "utf8")
- )
- reply = __network_manager.get(req)
- reply.finished.connect(pause.quit)
- is_ok = [True]
 
- def onError(self):
- is_ok[0] = False
- pause.quit()
+def remote_open_from_qgis(
+ uri: str,
+ headers: Dict[bytes, bytes] = {
+ b"Accept": b"application/xml",
+ b"Accept-Language": bytes(settings.value("default_language", "fr"), "utf8"),
+ b"User-Agent": bytes(settings.value("http_user_agent", __title__), "utf8"),
+ },
+) -> BytesIO:
+ """Opens a remote URL using Network Acess Manager. In fact, just a shortcut.
 
- reply.error.connect(onError)
- pause.exec_()
- return reply, is_ok[0]
+ :param uri: URI to request
+ :type uri: str
+ :param headers: HTTP headers. Defaults to: \
+ .. code-block:: python
 
+ {
+ b"Accept": b"application/xml",
+ b"Accept-Language": bytes(settings.value("default_language", "fr"), "utf8"),
+ b"User-Agent": bytes(settings.value("http_user_agent", __title__), "utf8")
+ }
+ :type headers: Dict[bytes, bytes], optional
 
-def remote_open_from_qgis(uri):
- """Opens a remote URL using QGIS proxy preferences"""
- reply, is_ok = _sync_get(QUrl.fromEncoded(bytes(uri, "utf8")))
- if not is_ok:
- raise RuntimeError("Network problem when downloading {}".format(uri))
- redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
- # Handle HTTP 302 redirections
- while redirect is not None and not redirect.isEmpty():
- reply, is_ok = _sync_get(redirect)
- if not is_ok:
- raise RuntimeError("Network problem when downloading {}".format(uri))
- redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
- r = bytes(reply.readAll())
- reply.close()
- return BytesIO(r)
+ :return: response content as bytesarray or None if something went wrong
+ :rtype: BytesIO
+ """
+ nam = NetworkAccessManager()
+ try:
+ response, content = nam.request(url=uri, headers=headers)
+ plg_logger.log(response.status_code)
+ return BytesIO(content)
+ except RequestsException as err:
+ logger.error(err)
+ plg_logger.log(
+ message="Request to {} failed. Trace: {}".format(uri, err),
+ log_level=2,
+ push=1,
+ )
+ return None

gml_application_schema_toolbox/core/settings.py

@@ -1,25 +1,39 @@
+#! python3 # noqa: E265
+
+# ############################################################################
+# ########## Imports ###############
+# ##################################
+
+# Standard library
 import os
 
-from qgis.PyQt.QtCore import QSettings
+# PyQGIS
+from qgis.core import QgsSettings
 
+# Project
 from gml_application_schema_toolbox.__about__ import __title__
 
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
 DEFAULT_GMLAS_CONFIG = os.path.realpath(
  os.path.join(os.path.dirname(__file__), "..", "conf", "gmlasconf.xml")
 )
 
 defaults = {
- "default_maxfeatures": 100,
- "wfs2_services": [],
- "default_wfs2_service": None,
- "default_import_method": "gmlas",
+ "debug_mode": False,
+ "default_access_mode": None,
+ "default_db_type": "SQLite",
  "default_gmlas_config": DEFAULT_GMLAS_CONFIG,
+ "default_import_method": "gmlas",
  "default_language": "en",
- "default_db_type": "SQLite",
- "default_access_mode": None,
+ "default_maxfeatures": 100,
+ "default_wfs2_service": None,
+ "wfs2_services": [],
 }
 
-settings = QSettings()
+settings = QgsSettings()
 settings.beginGroup(__title__)
 for key, value in defaults.items():
  if not settings.contains(key):

gml_application_schema_toolbox/gui/gmlas_panel_mixin.py

@@ -31,7 +31,7 @@
 
 # project package
 from gml_application_schema_toolbox.__about__ import __title__
-from gml_application_schema_toolbox.core.log_handler import PluginLogHandler
+from gml_application_schema_toolbox.toolbelt.log_handler import PlgLogger
 from gml_application_schema_toolbox.core.proxy import qgis_proxy_settings
 
 # ############################################################################
@@ -42,7 +42,7 @@
 class GmlasPanelMixin:
  def __init__(self):
  # map to the plugin log handler
- self.plg_logger = PluginLogHandler()
+ self.plg_logger = PlgLogger()
 
  @pyqtSlot()
  def on_gmlasConfigButton_clicked(self):

gml_application_schema_toolbox/gui/import_gmlas_panel.py

@@ -33,7 +33,7 @@
 from qgis.utils import iface
 
 from gml_application_schema_toolbox.__about__ import __title__
-from gml_application_schema_toolbox.core.log_handler import PluginLogHandler
+from gml_application_schema_toolbox.toolbelt.log_handler import PlgLogger
 from gml_application_schema_toolbox.core.proxy import qgis_proxy_settings
 from gml_application_schema_toolbox.core.settings import settings
 from gml_application_schema_toolbox.gui import InputError
@@ -54,7 +54,7 @@ def __init__(self, parent=None, gml_path=None):
  self.setupUi(self)
  self.databaseWidget.set_accept_mode(QFileDialog.AcceptSave)
  # map to the plugin log handler
- self.plg_logger = PluginLogHandler()
+ self.plg_logger = PlgLogger()
 
  self.gmlasConfigLineEdit.setText(settings.value("default_gmlas_config"))
  self.acceptLanguageHeaderInput.setText(settings.value("default_language"))

gml_application_schema_toolbox/gui/load_wizard.py

@@ -1,17 +1,32 @@
+#! python3 # noqa: E265
+
+# ############################################################################
+# ########## Imports ###############
+# ##################################
+
+# Standard library
 import os
 from tempfile import NamedTemporaryFile
 
+# PyQGIS
 from PyQt5 import uic
 from qgis.PyQt.QtCore import pyqtSlot
 from qgis.PyQt.QtWidgets import QFileDialog, QVBoxLayout, QWizard, QWizardPage
 
-from ..core.qgis_urlopener import remote_open_from_qgis
-from ..core.settings import settings
+# project
+from gml_application_schema_toolbox.core.settings import settings
+from gml_application_schema_toolbox.toolbelt.file_downloader import get_from_http
+from gml_application_schema_toolbox.toolbelt.log_handler import PlgLogger
+
 from .import_gmlas_panel import ImportGmlasPanel
 from .load_wizard_wfs import LoadWizardWFS
 from .load_wizard_xml import LoadWizardXML
 from .wait_cursor_context import WaitCursor
 
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
 PAGE_1_W, _ = uic.loadUiType(
  os.path.join(os.path.dirname(__file__), "..", "ui", "load_wizard_data_source.ui")
 )
@@ -21,15 +36,21 @@
 )
 
 PAGE_ID_DATA_SOURCE = 0
-PAGE_ID_WFS = 1
+PAGE_ID_GMLAS = 4
 PAGE_ID_LOADING = 2
+PAGE_ID_WFS = 1
 PAGE_ID_XML = 3
-PAGE_ID_GMLAS = 4
+
+
+# ############################################################################
+# ########## Classes ###############
+# ##################################
 
 
 class LoadWizardDataSource(QWizardPage, PAGE_1_W):
  def __init__(self, parent=None):
  super().__init__(parent)
+ self.log = PlgLogger().log
  self.setupUi(self)
 
  last = settings.value("last_source")
@@ -39,6 +60,8 @@ def __init__(self, parent=None):
  self.gmlPathLineEdit.setText(
  settings.value("last_file", settings.value("last_downloaded_file", ""))
  )
+ if __debug__:
+ self.log(message=f"DEBUG {__name__} loaded.", log_level=5)
 
  def nextId(self):
  if self.sourceFromWFS.isChecked():
@@ -69,12 +92,16 @@ def on_gmlPathButton_clicked(self):
  settings.setValue("last_file", filepath)
  self.gmlPathLineEdit.setText(filepath)
 
- def download(self, output_path):
+ def download(self, output_path: str):
+ """Download (if gmlPath is a HTTP URL) or copy a local file to tha output path.
+
+ :param output_path: output filepath
+ :type output_path: str
+ """
  input_path = self.gmlPathLineEdit.text()
  if input_path.startswith("http://") or input_path.startswith("https://"):
  # URL
- with open(output_path, "wb") as out:
- out.write(remote_open_from_qgis(input_path).read())
+ get_from_http(uri=input_path, output_path=output_path)
  else:
  # copy file
  with open(input_path, "rb") as inp: