import { Component } from '../../component/component.js';
import { LoadableComponent } from '../component/loadable_component.js';
import { BuildableComponent } from '../component/buildable_component.js';
import { InfoComponent } from '../../component/info_component.js';
import { v4 as uuid } from '../../../node_modules/uuid/dist/esm-browser/index.js';
import { checkPropTypes, copyProps, UUIDRegex, unique } from '../../lib.js';
import { InformationSource } from '../../reporter/information_source.js';
import { Reporter } from '../../reporter/reporter.js';
import { LoadingBase } from '../loader/loading_base.js';
/**
* @typedef {Object} BuildRequest
* @property {BuildableComponent} component
* @property {ComponentPart} part
* @property {LoadingQuality} quality
*/
/**
* @typedef {Object} BaseLoadTracker
* @property {LoadingBase} base
* @property {Object.<UUID,Promise<null>>} loads - When the promise resolves it returns a call to this._loadNext()
* @property {number} maxConcurrentLoads
*/
/**
* Component Loader schedules and loads component sources. If no Loading Bases are given
* it creates 1 Loading Base called "autogen" with location.origin as URL.
* Scheduling works in 3 steps:
* - From the desired Buildable Components calculated an ordered list of required Loadable Components
* - Omit the Loadable Components that are already (being) loaded
* - Start at the top of the Loadables list and load
* @extends {InformationSource}
*/
class ComponentLoader extends InformationSource {
/**
* @param {Reporter} reporter
* @param {Object} settings
* @param {UUID} [settings.id]
* @param {string} [settings.name]
* @param {Array<LoadingBase>} [settings.bases]
* @param {LoadingQuality} [settings.defaultLoadingQuality='medium']
* @param {number} [settings.concurrentLoadsPerBase = 2]
*/
constructor(reporter, settings = {}) {
super(reporter, settings);
checkPropTypes(
settings,
{},
{
bases: (val) =>
Array.isArray(val) &&
val.every((entry) => entry instanceof LoadingBase),
concurrentLoadsPerBase: 'number',
// defaultLoadingQuality: val => ['low', 'medium', 'high'].includes(val)
defaultLoadingQuality: (val) =>
Component.qualities.includes(val),
}
);
// if no loading bases are specified by the user
// create a new one here based on the location of the script
if (settings.bases) {
this.loadingBases = settings.bases;
} else {
const origin = location.origin;
this.report({
msg: `No loading bases specified, auto generating from origin ${origin}`,
});
this.loadingBases = [
new LoadingBase(reporter, {
for: ['default'],
url: new URL(origin),
}),
];
}
if (settings.concurrentLoadsPerBase) {
this._concurrentLoadsPerBase = settings.concurrentLoadsPerBase;
}
// default is medium
if (settings.defaultLoadingQuality) {
this._defaultLoadingQuality = settings.defaultLoadingQuality;
}
}
/**
* Presumed number of loading sockets per browser
* @static
* @type {Object.<string,number>}
*/
static socketTable = {
IE: 8,
Safari: 4,
Firefox: 6,
Opera: 6,
Chrome: 4,
};
/**
* @type {LoadingQuality}
* @default
*/
_defaultLoadingQuality = 'medium';
set defaultLoadingQuality(quality) {
this._defaultLoadingQuality = quality;
}
/**
* Component loads can be distributed over different bases
* to prevent/minimize browser throttling
* @type {Array<LoadingBase>}
* @protected
*/
_loadingBases;
get loadingBases() {
return this._loadingBases;
}
set loadingBases(newBases) {
if (this._buildQueue.length > 0) {
throw new Error(
`Can't change loading bases while loading resources`
);
}
this._loadingBases = newBases;
this._currentLoads = this.loadingBases.map((loadingBase) => ({
base: loadingBase,
loads: {},
maxConcurrentLoads: this._concurrentLoadsPerBase,
}));
}
/**
* Number of files that can be requested simultaneously per loading base.
* The assumption here is that the maximum number of concurrent loads
* depends on the browser, not on the host/base and therefore
* one value suffices, default is 2.
* @link http://www.websiteoptimization.com/speed/tweak/parallel/
* @link https://stackoverflow.com/questions/985431/max-parallel-http-connections-in-a-browser
* @type {number}
* @protected
* @default
*/
_concurrentLoadsPerBase = 2;
/**
* Queue of BuildableComponents in the order in which they should be loaded/built.
* By calling the .build() method, components can be added to this queue.
* @type {Array<BuildRequest>}
* @protected
*/
_buildQueue = [];
/**
* Queue of LoadableComponents in the order in which they should be loaded to
* deliver on the BuildRequests in the _buildQueue in the right order and as
* quickly as possible
* @type {Array<LoadableComponent>}
* @protected
*/
_loadableQueue = [];
/**
* Register of the current loads per loading base
* @type {Array<BaseLoadTracker>}
*/
_currentLoads = [];
/**
* Attempts to start loads on as many as possible loadable components from
* the _loadableQueue by distributing them over the loading bases in
* _currentLoads. The method is called automatically when a new buildable
* component is added to the _buildQueue[] and whenever a load is finished.
* @private
* @method
*/
_loadNext() {
// any items to load?
const nrOfLoadables = this._loadableQueue.length;
// bookkeeping the number of loads that will be initiated
let loadsAssigned = 0;
this.report({
msg: `Load next: ${
nrOfLoadables > 0
? nrOfLoadables + ' in queue.'
: 'Queue empty.'
}`,
level: 'info',
});
if (nrOfLoadables > 0) {
// iterate over the available loading bases to find an empty "slot"
for (let baseLoadTracker of this._currentLoads) {
let loadAssigned = false;
if (this._debug) {
console.debug(
baseLoadTracker.base.label,
'current loads:',
Object.keys(baseLoadTracker.loads).length
);
}
// for ( let loadableCandidate of this._loadableQueue) {
for ( let i = 0 ; i < this._loadableQueue.length ; i += 1 ) {
const loadableCandidate = this._loadableQueue[i];
// console.log('Next loadable candidate:', loadableCandidate.label );
// console.log(loadableCandidate)
// Loads are "stored" in the form of a promise in the .loads property
// of the BaseLoadTracker. When the load is completed the property
// is removed from the object. So, the number of entries in the object
// is always a current indication of the number of active loads.
// console.log(baseLoadTracker.base.for);
if ( loadableCandidate.loadingBase ) {
// this loading candidate is assigned to a specific loading base type
// if ( ! baseLoadTracker.for ) {
// continue;;
// }
// console.log('has loading base', loadableCandidate.loadingBase);
if (! baseLoadTracker.base.for.includes(loadableCandidate.loadingBase)) {
// console.log('skip')
continue;
}
// console.log('ok');
}
else if ( ! baseLoadTracker.base.for.includes("default")) {
continue;
}
// console.log('im here')
// if (baseLoadTracker.base.loadableComponentTypes) {
// const lct =
// baseLoadTracker.base.loadableComponentTypes;
// if (!lct.includes(this._loadableQueue[0].constructor)) {
// console.debug(
// `Loading base does not support type ${this._loadableQueue[0].constructor.name}`
// );
// continue;
// }
// }
if (
Object.keys(baseLoadTracker.loads).length <
baseLoadTracker.maxConcurrentLoads &&
this._loadableQueue.length > 0
) {
loadsAssigned += 1;
// Shift (remove) the "next" LoadableComponent entry from the
// queue and prepare loading it.
const nextComponentToLoad = loadableCandidate;
this._loadableQueue.splice(i,1);
// console.log('Loading', nextComponentToLoad);
this.report({
msg: `Free slot in base ${baseLoadTracker.base.url.href}, start ${this._defaultLoadingQuality} quality loading ${nextComponentToLoad.label}`,
level: 'debug',
});
// Assign the load process an id so the Promise can later
// be deleted from the object
const loadId = uuid();
// Initiate the load by calling the .load() method on the
// loadable component. This call will (usually) reference
// the .load method in the LoadableComponent base class
// which will in turn call the ._load() method in the
// extending class.
// When the load has finished remove the entry from the
// bookkeeping object (regardless of the success or failure
// of the load) and start a new loading distribution cycle
// by calling .loadNext(). Returnec for tail call optimization.
const loadPromise = nextComponentToLoad
.load(baseLoadTracker.base, this._defaultLoadingQuality)
.finally(() => {
delete baseLoadTracker.loads[loadId];
this.report({
msg: `Load process ${nextComponentToLoad.label} finished, calling loadNext`,
level: 'debug',
});
return this._loadNext();
});
// Strore the promise in the tracker for bookkeeping
baseLoadTracker.loads[loadId] = loadPromise;
loadAssigned = true;
break;
}
if (loadAssigned === true ) {
next;
}
}
}
// If there are components to be loaded, but no empty loading
// slots, simply wait for slots to free up, but waiting for
// loading promises to fulfill
if (loadsAssigned === 0) {
// console.warn(this._currentLoads);
this.report({
msg: 'Could not assign loads to bases, all slots are taken.',
level: 'debug',
});
}
}
}
// buildFlag = false;
// buildReg = [];
findLoadingBaseFor(type){
const base = this._loadingBases.find(base => base.for.includes(type));
if ( ! base ){
throw new Error(`No loading base for "${type}" found.`);
}
return base;
}
/**
* @method
* @param {BuildableComponent} component
* @param {ComponentPart} [part = 'main']
* @param {LoadingQuality} [quality = 'medium']
* @param {Boolean} [highPriority=false]
* @returns {Promise<BuildableComponent>}
*/
async build(
component,
part = 'main',
quality = 'medium',
highPriority = false
) {
//console.log( part )
// console.log('build command ', component.label, quality)
// if ( this.buildFlag === true ) {
// this.buildReg.push([component,part,quality,highPriority]);
// console.log('deferred');
// return;
// }
// this.buildFlag = true;
// /** @type {function} */
// let buildCommandProcessed = () => '';
// let buildCommandProcessedPromise = new Promise( res => buildCommandProcessed = res);
// const currentBuildCommandChain = this.buildFlag;
// currentBuildCommandChain.then( () => buildCommandProcessedPromise);
// await currentBuildCommandChain;
// console.log('accept command ', component.label)
this.report({
msg: `Build request for ${quality} quality ${part} part ${component.label}`,
level: 'info',
});
// if the component is already loaded simply return it
// wrapped in a promise
if (component.status[part][quality] === 'ready') {
this.report({ msg: `Component already ready, returning` });
// console.log(component)
return Promise.resolve(component);
}
// prepare a promise to return
// when all its dependencies are loaded, the component
// will update its own status, so the promise can simply
// reflect that status change
const loadCompleted = new Promise((resolve, reject) => {
const statusChangeHandler = ({
status,
quality: updateQuality,
part: updatePart,
}) => {
//console.log( part )
// console.warn(component.label, part, quality, status);
if (updateQuality === quality && updatePart === part) {
if (component._debug === true) {
console.warn(component.label, part, quality, status);
console.log(
JSON.parse(
JSON.stringify(component.dependencyStatus)
)
);
}
switch (status) {
case 'loading':
case 'partially-loaded':
// return here, so the event listener isn't removed yet
return;
case 'ready':
resolve(component);
break;
case 'error':
reject(component.error[part][quality]);
break;
default:
throw new Error(
'Component Loader unable to handle new component status ' +
status
);
}
if (component.hasListener('status-change', this.id)) {
component.removeListener('status-change', this.id);
}
}
};
component.on('status-change', statusChangeHandler, this.id);
}).finally(() => {
// remove build request
const cleanedUpBuildQueue = this._buildQueue.filter(
(br) =>
br.component !== component ||
br.part !== part ||
br.quality !== quality
);
const cleanupAmount =
this._buildQueue.length - cleanedUpBuildQueue.length;
if (cleanupAmount > 0) {
this.report({
msg: `Cleaned up ${cleanupAmount} old build requests`,
});
this._buildQueue = cleanedUpBuildQueue;
}
});
/** @type {BuildRequest} */
const buildRequest = {
component,
part,
quality,
};
//console.log( buildRequest )
const existingentry = this._buildQueue.find(
(br) =>
br.component === component &&
br.part === part &&
br.quality === quality
);
//console.log( existingentry )
if (existingentry !== undefined) {
// console.log('already in queue')
return loadCompleted;
}
// add the build request to the build queue
if (highPriority !== true) {
this._buildQueue.push(buildRequest);
} else {
this._buildQueue = [buildRequest, ...this._buildQueue];
}
// console.log('build queue length', this._buildQueue.length);
// immediately build the dependencies that have no dependencies themselves
// by calling their .autoUpdateStatus() method
const allRelevantComponents = [component, ...component.allDependencies];
//console.log( 'allRelevantComponents', allRelevantComponents.length)
//console.log( allRelevantComponents )
//console.log( allRelevantComponents.map( c => c.status.main))
const depLessBCDeps = allRelevantComponents.filter(
(dep) =>
// (dep instanceof InfoComponent)
// ||
dep instanceof BuildableComponent &&
dep.dependencies &&
Object.values(dep.dependencies[part]).length === 0
);
//console.log( depLessBCDeps )
const silentlyReadyDeps = allRelevantComponents.filter(
(dep) =>
dep instanceof BuildableComponent &&
dep.dependencies &&
Object.values(dep.dependencies[part]).every(
(subDep) =>
Array.isArray(subDep)
? subDep.find(
(altSubDep) =>
altSubDep.status[part][quality] ===
'ready'
) // one of the alternatives
: subDep.status[part][quality] === 'ready' // the thing itself
)
);
//console.log( silentlyReadyDeps )
const synchronouslyBuildables = [
...depLessBCDeps,
...silentlyReadyDeps,
].filter(unique);
//console.log(synchronouslyBuildables)
// console.log(synchronouslyBuildables.filter(sbb => sbb.status[part][quality] === 'not-loaded' ))
const laggingBuilds = synchronouslyBuildables
.filter((sbb) => sbb.status[part][quality] === 'not-loaded')
.map((sbb) => sbb.autoUpdateStatus());
//console.log( laggingBuilds )
// for ( let laggingBuild of laggingBuilds) {
// console.log('BUILD', laggingBuild.label)
// await laggingBuild.autoUpdateStatus();
// }
// this results in issues when multiple calls to .build are done in (close) succession
// await Promise.all(laggingBuilds);
// console.log('laggin done')
// reschedule
this._rescheduleLoadingOrder();
// return promise
// console.log('build command processed', this._loadableQueue)
// this.buildFlag = false;
// setTimeout(
// () => {
// if ( this.buildReg.length > 0 ) {
// const nextBuild = this.buildReg.splice(0,1);
// this.build
// }
// }
// )
// return Promise.all(laggingBuilds).then(() => {
// console.log('laggings')
// return loadCompleted
// });
return Promise.all(laggingBuilds).then(() => loadCompleted);
}
/**
* @private
*/
_qualitiesLowToHigh = Component.sortQualityArray(
Component.qualities
).reverse();
/**
* @protected
* @method
*/
_rescheduleLoadingOrder() {
// important to note here is that UI chains can not exist i.e.
// ok: main -> main -> main
// ok: UI -> main -> main
// not ok: UI -> UI -> main
/** @type {Object<LoadingQuality,Array<Component>>} */
const loadingOrderPerQuality = {};
const startQueueLength = this._loadableQueue.length;
const startBuildQueue = this._buildQueue.length;
for (let quality of Component.qualities) {
const singleQualityQueue = this._buildQueue.filter(
(entry) => entry.quality === quality
);
/**
* Component list with UI dependencies inlined
* @type {Array<Component>}
*/
const compListwUIDepsInl = [];
//TO DO HIER EEN FIX GENERIEKE PATRS!!
for (let buildRequest of singleQualityQueue) {
if (buildRequest.part === 'UI') {
if (buildRequest.component.dependencies.UI) {
compListwUIDepsInl.push(
...Object.values(
buildRequest.component.dependencies.UI
)
);
}
} else if (buildRequest.part === 'specs') {
if (buildRequest.component.dependencies.specs) {
compListwUIDepsInl.push(
...Object.values(
buildRequest.component.dependencies.specs
)
);
}
} else {
compListwUIDepsInl.push(buildRequest.component);
}
}
if (compListwUIDepsInl.length > 0) {
loadingOrderPerQuality[quality] =
ComponentLoader.optimalLoadingOrder(
compListwUIDepsInl,
quality
);
}
}
this._loadableQueue = this._qualitiesLowToHigh
.map(
(quality) =>
loadingOrderPerQuality[quality]?.relevantDependencies || []
)
.flat();
const endQueueLength = this._loadableQueue.length;
const endBuildQueue = this._buildQueue.length;
this.report({
msg: `Rescheduled loading order, ${startQueueLength} => ${endQueueLength}, ${startBuildQueue} - ${endBuildQueue}`,
});
this._loadNext();
}
/**
* Determines the optimal loading order for an array of components that
* need to be loaded at the same quality, only taking into account the 'main'
* part of the component.
* @static
* @param {Array<Component>} components
* @param {LoadingQuality} [quality='medium']
*/
static optimalLoadingOrder(components, quality = 'medium') {
// make a list of lists of needed loadable components
const dependencySets =
ComponentLoader.getLoadableDependencies(components);
// console.log( 'dependencySets', dependencySets)
// dependencies that are loaded or currently loading should be ignored
const relevantStates = ['not-loaded', 'partially-loaded'];
// return the one with the smallest expected size
const sizedDepSets = dependencySets.map((depSet) => {
// Set to array
const dependencies = [...depSet];
// filter relevant deps
const relevantDependencies = dependencies.filter((dep) =>
relevantStates.includes(dep.status.main[quality])
);
// console.log( relevantDependencies)
// console.log( relevantDependencies.forEach(dep => console.log( dep.label, dep.status.main[quality])))
return {
dependencies,
relevantDependencies,
sizeBytes: relevantDependencies.reduce((size, dep) => {
if (relevantStates.includes(dep.status.main[quality])) {
return size + dep.source[quality].sizeBytes;
} else {
// dependency is already loaded or currently loading
return size;
}
}, 0),
};
});
const smallestDepSet = sizedDepSets.sort(
(a, b) => a.sizeBytes - b.sizeBytes
)[0];
// console.log(sizedDepSets, 'smallest', smallestDepSet);
return smallestDepSet;
// components subscribe to load events from their dependencies and set themselves "usable" when
// all dependencies are loaded, so the responsibility for state control is still
// within the component
// however, as a component can't easily look "up" in the tree, it won't know
// which loading alternative to pick if there are multiple loading paths
}
/**
* Builds sets of dependency paths, only taking into account the main part of components
* @static
* @param {(Component|Array<Component>)} stuffToLoad
* @param {Array<Set>} [depSetsArr]
*/
static getLoadableDependencies(
stuffToLoad,
/* out */ depSetsArr = [new Set()],
/* out */ analysedComponents = []
) {
// console.log( 'getLoadableDependencies', stuffToLoad.constructor.name, stuffToLoad instanceof LoadableComponent, stuffToLoad instanceof BuildableComponent)
if (stuffToLoad instanceof LoadableComponent) {
// console.log( 'Adding loadable:', stuffToLoad.label)
depSetsArr.forEach((set) => set.add(stuffToLoad));
//console.log(depSetsArr)
} else {
let componentList = null;
if (stuffToLoad instanceof BuildableComponent) {
if (!stuffToLoad.dependencies) {
console.log(stuffToLoad);
}
componentList = Object.values(stuffToLoad.dependencies.main);
// console.log( 'BC', componentList)
} else if (
Array.isArray(stuffToLoad) &&
stuffToLoad.every((entry) => entry instanceof Component)
) {
componentList = stuffToLoad;
// console.log( 'ARR', componentList)
} else if (stuffToLoad instanceof InfoComponent) {
componentList = [];
// console.log( 'Info', componentList)
} else {
// console.warn(stuffToLoad);
throw new Error(
'Cannot get dependencies of ' + typeof stuffToLoad
);
}
for (let component of componentList) {
if (component instanceof Component) {
if (analysedComponents.indexOf(component) === -1) {
// console.log( 'Analysing', stuffToLoad.label)
// components in the analysedComponents list will not be analysed again
// therefore, only non-alternative
analysedComponents.push(stuffToLoad);
depSetsArr = this.getLoadableDependencies(
component,
depSetsArr,
analysedComponents
);
} else {
// console.log(`Already analysed ${component.label} - skipping`);
// console.log(analysedComponents, analysedComponents.indexOf(component))
continue;
}
} else if (Array.isArray(component)) {
const newDepSetsArr = [];
// console.log( 'Array alternatives', component.length)
for (let subDep of component) {
if (analysedComponents.indexOf(subDep) === -1) {
const depSetsArrClone = depSetsArr.map(
(depSet) => new Set(depSet)
);
newDepSetsArr.push(
...this.getLoadableDependencies(
subDep,
depSetsArrClone,
analysedComponents.slice()
)
);
} else {
// console.log(`Already analysed ${subDep.label} - skipping`);
continue;
}
}
if (newDepSetsArr.length > 0) {
depSetsArr = newDepSetsArr;
}
}
}
}
return depSetsArr;
}
// /**
// * @static
// * @param {LoadableComponent} component
// */
// static getLoader(component) {
// switch (component.constructor.name) {
// case 'WrappedImage':
// return ComponentLoader.imageLoader;
// case 'GeometryFile':
// return ComponentLoader.GLTFLoader;
// case 'JSONFile':
// return ComponentLoader.fileLoader;
// default:
// throw new Error('Unknown component type');
// }
// }
}
export { ComponentLoader };