Implement new library loading approach

.gitattributes

@@ -1,2 +1,3 @@
 # Ignore all Git auto CR/LF line endings conversions
-* binary
+* -text
+pyproject.toml export-subst

.travis.yml

@@ -16,7 +16,7 @@ python:
   - "3.9"
 
 # Scripts to run at install stage
-install: ./configure
+install: ./configure --dev
 
 # Scripts to run at script stage
 script: tmp/bin/pytest -vvs -n 2

CHANGELOG.rst

@@ -4,6 +4,18 @@ Release notes
 vNext
 -----
 
+Version 21.5.30
+---------------
+
+- Update vendored pygments to 2.9.0
+- Update commoncode to latest version
+- Use new libmagic configuration based on a plugin, and environment variable
+  or the system path. 
+- Only do content-based detection for programing language if there is no extension.
+- Remove Actionscript3 from programing language detection. This is not common
+  enough and is the source of too many false positives.
+
+
 
 Version 21.2.24
 ---------------

NOTICE

@@ -1,19 +1,8 @@
 #
-# Copyright (c) nexB Inc. and others.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Visit https://aboutcode.org and https://github.com/nexB/ for support and download.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # ScanCode is a trademark of nexB Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/typecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #

README.rst

@@ -12,6 +12,11 @@ macOS) and Pygments. It started as library in scancode-toolkit.
 Visit https://aboutcode.org and https://github.com/nexB/ for support and download.
 
 
+We run CI tests on:
+
+ - Travis https://travis-ci.org/github/nexB/typecode
+ - Azure pipelines https://dev.azure.com/nexB/typecode/_build
+
 To install this package with its full capability (where the binaries for
 libmagic are installed), use the `full` option::
 
@@ -22,10 +27,39 @@ system, use the `minimal` option::
 
     pip install typecode
 
+In this case, you will need to provide a working libmagic and its database
+available in one of these ways:
+
+- **a typecode-libmagic plugin**: See the standard ones at 
+  https://github.com/nexB/scancode-plugins/tree/main/builtins
+  These can either bundle a libmagic library and its magic database or expose a
+  system-installed libmagic.
+  They do so by providing a plugin entry point as a ``scancode_location_provider``
+  for ``typecode_libmagic`` which points to a callable that must return a mapping
+  with these two keys:
+
+    - 'typecode.libmagic.dll': the absolute path to a libmagic DLL
+    - 'typecode.libmagic.db': the absolute path to a libmagic 'magic.mgc' database
+
+  See for example:
+
+    - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/typecode_libmagic-linux/setup.py#L42
+    - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/typecode_libmagic-linux/src/typecode_libmagic/__init__.py#L32
+
+- **environment variables**:
+
+  - TYPECODE_LIBMAGIC_PATH: the absolute path to a libmagic DLL
+  - TYPECODE_LIBMAGIC_DB_PATH: the absolute path to a libmagic 'magic.mgc' database
+
+- **a system-installed libmagic and its database availale in the system PATH**:
+
+
+The supported libmagic version is 5.39.
+
 
 To set up the development environment::
 
-    source configure
+    source configure --dev
 
 To run unit tests::
 
@@ -35,3 +69,13 @@ To clean up development environment::
 
     ./configure --clean
 
+
+To update Pygment to a newer vendored version use vendy:
+
+  - Update the version of pygments in ``pyproject.toml``
+  - Run ``vendy``
+  - Update the src/typecpde/pygments_lexers_mapping.py
+    and src/typecode/pygments_lexers.py scripts accordingly, including their
+    ABOUT files
+
+

azure-pipelines.yml

@@ -7,39 +7,39 @@
 
 jobs:
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu16_cpython
           image_name: ubuntu-16.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -vvs
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython
           image_name: ubuntu-18.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1014_cpython
           image_name: macos-10.14
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1015_cpython
           image_name: macos-10.15

configure

@@ -1,44 +1,169 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) nexB Inc. and others.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/ for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #
 
 set -e
 #set -x
 
-# source this script for a basic setup and configuration for local development
+################################
+# A configuration script to set things up: 
+# create a virtualenv and install or update thirdparty packages.
+# Source this script for initial configuration
+# Use configure --help for details
+#
+# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
+# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
+################################
+CLI_ARGS=$1
 
-CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+################################
+# Defaults. Change these variables to customize this script
+################################
 
+# Requirement arguments passed to pip and used by default or with --dev.
+REQUIREMENTS="--editable .[full]"
+DEV_REQUIREMENTS="--editable .[full,testing]"
 
-if [[ "$1" == "--clean" ]]; then
-    rm -rf "$CONFIGURE_ROOT_DIR/tmp"
-    exit
+# where we create a virtualenv
+VIRTUALENV_DIR=tmp
+
+# Cleanable files and directories with the --clean option
+CLEANABLE="
+    build
+	tmp"
+
+# extra  arguments passed to pip
+PIP_EXTRA_ARGS=" "
+
+# the URL to download virtualenv.pyz if needed
+VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
+################################
+
+
+################################
+# Current directory where this script lives
+CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
+
+
+################################
+# Set the quiet flag to empty if not defined 
+if [[ "$CFG_QUIET" == "" ]]; then
+    CFG_QUIET=" "
 fi
 
 
-if [[ "$PYTHON_EXE" == "" ]]; then
-    PYTHON_EXE=python3
+################################
+# find a proper Python to run
+# Use environment variables or a file if available.
+# Otherwise the latest Python by default.
+if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+    # check for a file named PYTHON_EXECUTABLE 
+    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+    else
+        PYTHON_EXECUTABLE=python3
+    fi
 fi
 
 
-function setup {
-    # create a virtualenv on Python
-    mkdir -p $CONFIGURE_ROOT_DIR/tmp
-    wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz
-    $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp
-    source $CONFIGURE_ROOT_DIR/tmp/bin/activate
-    $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel
+################################
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
 }
 
 
-setup
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
 
-$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[testing] -e .[full]
 
-if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then
-    source "$CONFIGURE_ROOT_DIR/tmp/bin/activate"
-fi
+create_virtualenv() {
+    # create a virtualenv for Python
+    # Note: we do not use the bundled Python 3 "venv" because its behavior and
+    # presence is not consistent across Linux distro and sometimes pip is not
+    # included either by default. The virtualenv.pyz app cures all these issues.
+
+    VENV_DIR="$1"
+    if [ ! -f "$CFG_BIN_DIR/python" ]; then
+
+        mkdir -p "$CFG_ROOT_DIR/$VENV_DIR"
+
+        if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
+        else
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
+            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
+        fi
+
+        $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
+            --wheel embed --pip embed --setuptools embed \
+            --seeder pip \
+            --never-download \
+            --no-periodic-update \
+            --no-vcs-ignore \
+            $CFG_QUIET \
+            "$CFG_ROOT_DIR/$VENV_DIR"
+    fi
+}
+
+
+install_packages() {
+    # install requirements in virtualenv
+    # note: --no-build-isolation means that pip/wheel/setuptools will not
+    # be reinstalled a second time and reused from the virtualenv and this
+    # speeds up the installation.
+    # We always have the PEP517 build dependencies installed already.
+
+    "$CFG_BIN_DIR/pip" install \
+        --upgrade \
+        --no-build-isolation \
+        $CFG_QUIET \
+        $PIP_EXTRA_ARGS \
+        $1
+}
+
+
+################################
+# Main command line entry point
+CFG_DEV_MODE=0
+CFG_REQUIREMENTS=$REQUIREMENTS
+
+case "$CLI_ARGS" in
+    --help)    cli_help;;
+    --clean)   clean;;
+    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+esac
+
+create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$CFG_REQUIREMENTS"
+. "$CFG_BIN_DIR/activate"
 
 set +e