Updating processing the url and repo_url (#22)

* update (includes.py): path processing

* upgate (includes.py): update processing repo_url

* update: add test

* update (readme.md): add note for repo_url

* update readme and test

* resolve conflict

* update: deleted print

* fix: added error handling

* fix: full links

Co-authored-by: Екатерина Московская <ekaterina.markova@rt.ru>

Author: KateVishnya

README.md

@@ -110,6 +110,28 @@ Text below is taken from a remote repository on branch develop.
 <include repo_url="https://github.com/foo/bar.git" revision="develop" path="path/to/doc.md"></include>
 ```
 
+To include a text from HTTP(S) URL use the `url` attributes:
+
+```markdown
+Text below is taken from a remote repository on branch develop.
+
+<include url="https://github.com/foo/bar/path/to/doc.md"></include>
+```
+
+> **Note**
+>
+> For projects in GitHub, you must specify the full path to the raw file, while not specifying the file extension `.md`.
+    
+``` markdown
+<include url="https://github.com/path/to/doc/raw/master/doc" nohead="true"></include>
+```
+
+> For projects in GitLab, you must specify the full path to the raw file, and you must specify the file extension `.md`.
+
+```markdown
+<include url="https://gitlub.com/path/to/doc/raw/master/doc.md" nohead="true"></include>
+```
+
 To include a code snippet, use `wrap_code` and `code_language` attributes:
 
 ```markdown
@@ -118,8 +140,6 @@ wrap_code="triple_backticks" code_language="yaml">
 </include>
 ```
 
-
-
 #### Attributes
 
 `src`
@@ -134,13 +154,18 @@ wrap_code="triple_backticks" code_language="yaml">
 `path`
 :   Path to the file inside the remote Git repository.
 
-    >    **Note**
-    >
-    >    If you are using the new syntax, the `src` attribute is required to include a local file, `url` is required to include a remote file, and the `repo_url` and `path` attributes are required to include a file from a remote Git repository. All other attributes are optional.
+>    **Note**
+>    
+>    This parameter is required!
+        Its absence will lead to incorrect operation Foliant.
 
-    >    **Note**
-    >
-    >    Foliant 1.0.9 supports the processing of attribute values as YAML. You can precede the values of attributes by the `!path`, `!project_path`, and `!rel_path` modifiers (i.e. YAML tags). These modifiers can be useful in the `src`, `path`, and `project_root` attributes.
+>    **Note**
+>
+>    If you are using the new syntax, the `src` attribute is required to include a local file, `url` is required to include a remote file, and the `repo_url` and `path` attributes are required to include a file from a remote Git repository. All other attributes are optional.
+
+>    **Note**
+>
+>    Foliant 1.0.9 supports the processing of attribute values as YAML. You can precede the values of attributes by the `!path`, `!project_path`, and `!rel_path` modifiers (i.e. YAML tags). These modifiers can be useful in the `src`, `path`, and `project_root` attributes.
 
 `revision`
 :   Revision of the Git repository.

foliant/preprocessors/includes.py

@@ -74,6 +74,20 @@ def _find_file(
 
         return result
 
+   
+    def create_full_link(self, repo_url: str, revision: str, path: str):
+
+        if repo_url.endswith('.git'):
+            repo_url = repo_url[:-4]
+        
+        if revision:
+            full_repo_url=repo_url + '/tree/' + revision + '/' + path.rpartition('/')[0]
+            
+        else:
+            full_repo_url=repo_url + '/-/blob/master/' + path.rpartition('/')[0]
+           
+        return full_repo_url
+
     def _download_file_from_url(self, url: str) -> Path:
         '''Download file as the content of resource located at specified URL.
         Place downloaded file into the cache directory with a unique name.
@@ -107,7 +121,7 @@ def _download_file_from_url(self, url: str) -> Path:
 
             response = urllib.request.urlopen(url)
             charset = 'utf-8'
-
+            
             if response.headers['Content-Type']:
                 charset_match = re.search(r'(^|[\s;])charset=(?P<charset>[^\s;]+)', response.headers['Content-Type'])
 
@@ -117,10 +131,31 @@ def _download_file_from_url(self, url: str) -> Path:
             self.logger.debug(f'Detected source charset: {charset}')
 
             downloaded_content = response.read().decode(charset)
-
+                  
             self._downloaded_dir_path.mkdir(parents=True, exist_ok=True)
 
+            # The beginning of the block codes for converting relative paths to links
+            dict_new_link = {}
+            regexp_find_link = re.compile('\[.+?\]\(.+?\)')
+            regexp_find_path = re.compile('\(.+?\)')
+           
+            old_found_link = regexp_find_link.findall(downloaded_content)
+
+            for line in old_found_link:
+                exceptions_simbols = re.findall(r'http|@|:',line)
+                if exceptions_simbols:
+                    continue
+                else:
+                    relative_path = regexp_find_path.findall(line)
+                    sub_relative_path = re.findall(r'\[.+?\]', line)
+                    dict_new_link[line] = sub_relative_path[0] + '(' + url.rpartition('/')[0].replace('raw', 'blob')+'/'+ relative_path[0].partition('(')[2]
+            
+            for line in dict_new_link:
+                downloaded_content = downloaded_content.replace(line, dict_new_link[line])
+            # End of the conversion code block         
+
             with open(downloaded_file_path, 'w', encoding='utf8') as downloaded_file:
+                
                 downloaded_file.write(downloaded_content)
 
         else:
@@ -652,7 +687,8 @@ def _process_include(
         to_id: str or None = None,
         to_end: bool = False,
         sethead: int or None = None,
-        nohead: bool = False
+        nohead: bool = False,
+        include_link: str or None = None
     ) -> str:
         '''Replace a local include statement with the file content. Necessary
         adjustments are applied to the content: cut between certain headings,
@@ -677,10 +713,32 @@ def _process_include(
             f'Included file path: {included_file_path}, from heading: {from_heading}, ' +
             f'to heading: {to_heading}, sethead: {sethead}, nohead: {nohead}'
         )
-
+        
         with open(included_file_path, encoding='utf8') as included_file:
             included_content = included_file.read()
 
+             # The beginning of the block codes for converting relative paths to links
+            if include_link:
+                dict_new_link = {}
+                regexp_find_link = re.compile('\[.+?\]\(.+?\)')
+                regexp_find_path = re.compile('\(.+?\)')
+            
+                old_found_link = regexp_find_link.findall(included_content)
+
+                for line in old_found_link:
+                    exceptions_simbols = re.findall(r'http|@|:',line)
+                    if exceptions_simbols:
+                        continue
+                    else:
+                        relative_path = regexp_find_path.findall(line)
+                        sub_relative_path = re.findall(r'\[.+?\]', line)
+                        dict_new_link[line] = sub_relative_path[0] + '(' + include_link.rpartition('/')[0].replace('raw', 'blob')+'/'+ relative_path[0].partition('(')[2]
+                
+                for line in dict_new_link:
+                    included_content = included_content.replace(line, dict_new_link[line])
+            # End of the conversion code block         
+
+
             if self.config.get('escape_code', False):
                 if isinstance(self.config['escape_code'], dict):
                     escapecode_options = self.config['escape_code'].get('options', {})
@@ -736,7 +794,7 @@ def _process_include(
                 '!rel_path',
                 included_file_path.parent
             )
-
+           
         return included_content
 
     def process_includes(
@@ -926,8 +984,10 @@ def process_includes(
                             repo_path / options.get('project_root', '')
                         ).resolve()
 
-                        self.logger.debug(f'Set new current project root path: {current_project_root_path}')
+                        include_link = self.create_full_link(options.get('repo_url'), options.get('revision'), options.get('path'))
 
+                        self.logger.debug(f'Set new current project root path: {current_project_root_path}')
+                        
                         processed_content_part = self._process_include(
                             included_file_path=included_file_path,
                             project_root_path=current_project_root_path,
@@ -937,7 +997,8 @@ def process_includes(
                             to_id=options.get('to_id'),
                             to_end=options.get('to_end'),
                             sethead=current_sethead,
-                            nohead=options.get('nohead')
+                            nohead=options.get('nohead'),
+                            include_link=include_link
                         )
 
                     elif options.get('url'):
@@ -1094,7 +1155,7 @@ def apply(self):
             for source_file_path in self.working_dir.rglob(source_files_extension):
                 with open(source_file_path, encoding='utf8') as source_file:
                     source_content = source_file.read()
-
+                    
                 processed_content = self.process_includes(
                     source_file_path,
                     source_content,

test/test_includes.py

@@ -5,6 +5,7 @@
 from unittest import TestCase
 from foliant_test.preprocessor import PreprocessorTestFramework
 from .utils import data_file_content
+import urllib.request
 
 
 logging.disable(logging.CRITICAL)
@@ -28,6 +29,18 @@ def test_src(self):
             input_mapping=input_map,
             expected_mapping=expected_map,
         )
+    
+    def test_url(self):
+        input_map = {
+            'index.md': '# My title\n\n<include url="https://github.com/foliant-docs/foliantcontrib.includes/raw/master/LICENSE" nohead="true"></include>',
+        }
+        expected_map = {
+            'index.md': f'# My title\n\n{data_file_content("../LICENSE")}',
+        }
+        self.ptf.test_preprocessor(
+            input_mapping=input_map,
+            expected_mapping=expected_map,
+        )
 
     def test_repo_path(self):
         input_map = {
@@ -40,6 +53,18 @@ def test_repo_path(self):
             input_mapping=input_map,
             expected_mapping=expected_map,
         )
+        
+    def test_include_link(self):
+        input_map = {
+            'index.md': '# My title\n\n<include repo_url="https://github.com/foliant-docs/foliantcontrib.includes" revision="master" path="LICENSE"></include>',
+        }
+        expected_map = {
+             'index.md': f'# My title\n\n{data_file_content("../LICENSE")}',
+        }
+        self.ptf.test_preprocessor(
+            input_mapping=input_map,
+            expected_mapping=expected_map,
+        )
 
     def test_nohead(self):
         input_map = {