Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prebid Core: Library support #8527

Merged
merged 7 commits into from
Jun 14, 2022
Merged

Prebid Core: Library support #8527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ab3b71c
adds extra utils
karimMourra Jun 6, 2022
98ce2ea
imports from extra utils
karimMourra Jun 6, 2022
123c00e
adds libraries to .submodules
karimMourra Jun 6, 2022
b324d90
adds libraries to build process
karimMourra Jun 6, 2022
93b23f5
renames extraUtils
karimMourra Jun 7, 2022
6c66760
update userId submodules list
karimMourra Jun 9, 2022
4c48672
Merge branch 'master' into library-support
karimMourra Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 53 additions & 16 deletions gulpHelpers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ const MANIFEST = 'package.json';
const through = require('through2');
const _ = require('lodash');
const gutil = require('gulp-util');
const submodules = require('./modules/.submodules.json');
const dependencyMap = require('./modules/.submodules.json');
const submodules = dependencyMap.parentModules;
const libraries = dependencyMap.libraries;

const MODULE_PATH = './modules';
const LIBRARY_PATH = './libraries';
const BUILD_PATH = './build/dist';
const DEV_PATH = './build/dev';
const ANALYTICS_PATH = '../analytics';
Expand Down Expand Up @@ -68,34 +71,68 @@ module.exports = {
}
});

Object.keys(libraries).forEach(library => {
if (!modules.includes(library) && modules.some(module => libraries[library].dependants.includes(module))) {
modules.unshift(library);
}
});

return modules;
},
getParentLibraries(moduleName) {
const libraryNames = [];
Object.keys(libraries).forEach(libraryName => {
const library = libraries[libraryName];
if (library.dependants.includes(moduleName)) {
libraryNames.push(libraryName);
}
});
return libraryNames;
},
getLibraryFiles(name) {
const library = libraries[name];
const files = library.files.map(file => require.resolve(file, {paths: ['./libraries/' + name + '/']}));
return files;
},
isLibrary(name) {
return !!libraries[name];
},
getModules: _.memoize(function(externalModules) {
externalModules = externalModules || [];
var internalModules;
try {
var getInternalModules = function(absolutePath) {
return fs.readdirSync(absolutePath)
.filter(file => (/^[^\.]+(\.js)?$/).test(file))
.reduce((memo, file) => {
var moduleName = file.split(new RegExp('[.\\' + path.sep + ']'))[0];
var modulePath = path.join(absolutePath, file);
if (fs.lstatSync(modulePath).isDirectory()) {
modulePath = path.join(modulePath, 'index.js')
}
if (fs.existsSync(modulePath)) {
memo[modulePath] = moduleName;
}
return memo;
}, {});
};

var absoluteModulePath = path.join(__dirname, MODULE_PATH);
internalModules = fs.readdirSync(absoluteModulePath)
.filter(file => (/^[^\.]+(\.js)?$/).test(file))
.reduce((memo, file) => {
var moduleName = file.split(new RegExp('[.\\' + path.sep + ']'))[0];
var modulePath = path.join(absoluteModulePath, file);
if (fs.lstatSync(modulePath).isDirectory()) {
modulePath = path.join(modulePath, 'index.js')
}
if (fs.existsSync(modulePath)) {
memo[modulePath] = moduleName;
}
return memo;
}, {});
var absoluteLibraryPath = path.join(__dirname, LIBRARY_PATH);

internalModules = getInternalModules(absoluteModulePath);
var internalLibraries = getInternalModules(absoluteLibraryPath);
Object.assign(internalModules, internalLibraries);
} catch (err) {
internalModules = {};
}
return Object.assign(externalModules.reduce((memo, module) => {
try {
// prefer internal project modules before looking at project dependencies
var modulePath = require.resolve(module, {paths: ['./modules']});
if (modulePath === '') modulePath = require.resolve(module);
var modulePath = require.resolve(module, {paths: [MODULE_PATH, LIBRARY_PATH]});
if (modulePath === '') {
modulePath = require.resolve(module);
}

memo[modulePath] = module;
} catch (err) {
Expand Down
1 change: 1 addition & 0 deletions gulpfile.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ function lint(done) {
return gulp.src([
'src/**/*.js',
'modules/**/*.js',
'libraries/**/*.js',
'test/**/*.js',
'plugins/**/*.js',
'!plugins/**/node_modules/**',
Expand Down
11 changes: 11 additions & 0 deletions libraries/getOrigin/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/**
* Returns the origin
*/
export function getOrigin() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there any restrictions on library files? What happens if you want to import another file from the library file?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I initially envisioned this, the maintainer of the library would determine which files are necessary.
If users were to find the library too heavy, the maintainer could either split the library (i.e by feature or by light vs advanced) or a competitor could offer a lighter library.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'll probably need to cover this at a later date, with some best practices or boundaries

// IE10 does not have this property. https://gist.github.com/hbogs/7908703
if (!window.location.origin) {
return window.location.protocol + '//' + window.location.hostname + (window.location.port ? ':' + window.location.port : '');
} else {
return window.location.origin;
}
}
152 changes: 83 additions & 69 deletions modules/.submodules.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,71 +1,85 @@
{
"userId": [
"33acrossIdSystem",
"admixerIdSystem",
"adtelligentIdSystem",
"amxIdSystem",
"britepoolIdSystem",
"connectIdSystem",
"cpexIdSystem",
"criteoIdSystem",
"dacIdSystem",
"deepintentDpesIdSystem",
"dmdIdSystem",
"fabrickIdSystem",
"hadronIdSystem",
"id5IdSystem",
"ftrackIdSystem",
"identityLinkIdSystem",
"idxIdSystem",
"imuIdSystem",
"intentIqIdSystem",
"justIdSystem",
"kinessoIdSystem",
"liveIntentIdSystem",
"lotamePanoramaIdSystem",
"merkleIdSystem",
"mwOpenLinkIdSystem",
"naveggIdSystem",
"netIdSystem",
"novatiqIdSystem",
"parrableIdSystem",
"pubProvidedIdSystem",
"publinkIdSystem",
"quantcastIdSystem",
"sharedIdSystem",
"tapadIdSystem",
"trustpidSystem",
"uid2IdSystem",
"unifiedIdSystem",
"verizonMediaIdSystem",
"zeotapIdPlusIdSystem",
"adqueryIdSystem",
"gravitoIdSystem"
],
"adpod": [
"freeWheelAdserverVideo",
"dfpAdServerVideo"
],
"rtdModule": [
"airgridRtdProvider",
"akamaiDapRtdProvider",
"browsiRtdProvider",
"dgkeywordRtdProvider",
"geoedgeRtdProvider",
"hadronRtdProvider",
"haloRtdProvider",
"iasRtdProvider",
"jwplayerRtdProvider",
"medianetRtdProvider",
"optimeraRtdProvider",
"permutiveRtdProvider",
"reconciliationRtdProvider",
"sirdataRtdProvider",
"timeoutRtdProvider",
"weboramaRtdProvider"
],
"fpdModule": [
"enrichmentFpdModule",
"validationFpdModule"
]
"parentModules": {
"userId": [
"33acrossIdSystem",
"admixerIdSystem",
"adtelligentIdSystem",
"amxIdSystem",
"britepoolIdSystem",
"connectIdSystem",
"cpexIdSystem",
"criteoIdSystem",
"dacIdSystem",
"deepintentDpesIdSystem",
"dmdIdSystem",
"fabrickIdSystem",
"hadronIdSystem",
"id5IdSystem",
"ftrackIdSystem",
"identityLinkIdSystem",
"idxIdSystem",
"imuIdSystem",
"intentIqIdSystem",
"justIdSystem",
"kinessoIdSystem",
"liveIntentIdSystem",
"lotamePanoramaIdSystem",
"merkleIdSystem",
"mwOpenLinkIdSystem",
"naveggIdSystem",
"netIdSystem",
"novatiqIdSystem",
"parrableIdSystem",
"pubProvidedIdSystem",
"publinkIdSystem",
"quantcastIdSystem",
"sharedIdSystem",
"tapadIdSystem",
"trustpidSystem",
"uid2IdSystem",
"unifiedIdSystem",
"verizonMediaIdSystem",
"zeotapIdPlusIdSystem",
"adqueryIdSystem",
"gravitoIdSystem"
],
"adpod": [
"freeWheelAdserverVideo",
"dfpAdServerVideo"
],
"rtdModule": [
"airgridRtdProvider",
"akamaiDapRtdProvider",
"browsiRtdProvider",
"dgkeywordRtdProvider",
"geoedgeRtdProvider",
"hadronRtdProvider",
"haloRtdProvider",
"iasRtdProvider",
"jwplayerRtdProvider",
"medianetRtdProvider",
"optimeraRtdProvider",
"permutiveRtdProvider",
"reconciliationRtdProvider",
"sirdataRtdProvider",
"timeoutRtdProvider",
"weboramaRtdProvider"
],
"fpdModule": [
"enrichmentFpdModule",
"validationFpdModule"
]
},
"libraries": {
"getOrigin": {
"files": [
"./index.js"
],
"dependants": [
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in principle we should be able to calculate these automatically by looking at imports. I think webpack's chunking feature does it under the hood but I don't see an easy way to extract the graph for us to use in bundle.

We can easily add it later so no request for this PR, just noting it here in case you are familiar with that feature.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

makes sense, I'm happy to explore that in the near future when cycles free up

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 I could see this dependents list getting out of hand quickly.

"ooloAnalyticsAdapter",
"resetdigitalBidAdapter",
"rtbhouseBidAdapter.js"
]
}
}
}
3 changes: 2 additions & 1 deletion modules/ooloAnalyticsAdapter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { _each, deepClone, pick, deepSetValue, getOrigin, logError, logInfo } from '../src/utils.js';
import { _each, deepClone, pick, deepSetValue, logError, logInfo } from '../src/utils.js';
import { getOrigin } from '../libraries/getOrigin/index.js';
import adapter from '../src/AnalyticsAdapter.js'
import adapterManager from '../src/adapterManager.js'
import CONSTANTS from '../src/constants.json'
Expand Down
3 changes: 2 additions & 1 deletion modules/resetdigitalBidAdapter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

import { timestamp, deepAccess, getOrigin } from '../src/utils.js';
import { timestamp, deepAccess } from '../src/utils.js';
import { getOrigin } from '../libraries/getOrigin/index.js';
import { config } from '../src/config.js';
import {registerBidder} from '../src/adapters/bidderFactory.js';
const BIDDER_CODE = 'resetdigital';
Expand Down
3 changes: 2 additions & 1 deletion modules/rtbhouseBidAdapter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {deepAccess, getOrigin, isArray, logError} from '../src/utils.js';
import {deepAccess, isArray, logError} from '../src/utils.js';
import { getOrigin } from '../libraries/getOrigin/index.js';
import {BANNER, NATIVE} from '../src/mediaTypes.js';
import {registerBidder} from '../src/adapters/bidderFactory.js';
import {includes} from '../src/polyfill.js';
Expand Down
11 changes: 0 additions & 11 deletions src/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,17 +925,6 @@ export function getUserConfiguredParams(adUnits, adUnitCode, bidder) {
.filter((bidderData) => bidderData.bidder === bidder)
.map((bidderData) => bidderData.params || {});
}
/**
* Returns the origin
*/
export function getOrigin() {
// IE10 does not have this property. https://gist.github.com/hbogs/7908703
if (!window.location.origin) {
return window.location.protocol + '//' + window.location.hostname + (window.location.port ? ':' + window.location.port : '');
} else {
return window.location.origin;
}
}

/**
* Returns Do Not Track state
Expand Down
15 changes: 14 additions & 1 deletion webpack.conf.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,25 @@ module.exports = {
}
};
const selectedModules = new Set(helpers.getArgModules());

Object.entries(helpers.getModules()).forEach(([fn, mod]) => {
if (selectedModules.size === 0 || selectedModules.has(mod)) {
entry[mod] = {
const moduleEntry = {
import: fn,
dependOn: 'prebid-core'
};

if (helpers.isLibrary(mod)) {
const libraryFiles = helpers.getLibraryFiles(mod);
moduleEntry.import = libraryFiles || moduleEntry.import;
}

const libraries = helpers.getParentLibraries(mod);
if (libraries.length) {
moduleEntry.dependOn = ['prebid-core'].concat(libraries);
}

entry[mod] = moduleEntry;
}
});
return entry;
Expand Down