import { Component } from '../../component/component.js';
import { LoadableComponent } from '../component/loadable_component.js';
import { BuildableComponent } from '../component/buildable_component.js';
import { InfoComponent } from '../../component/info_component.js';
import { v4 as uuid } from '../../../node_modules/uuid/dist/esm-browser/index.js';
import { checkPropTypes, copyProps, UUIDRegex, unique } from '../../lib.js';
import { InformationSource } from '../../reporter/information_source.js';
import { Reporter } from '../../reporter/reporter.js';
import { LoadingBase } from '../loader/loading_base.js';
/**
* @typedef {Object} BuildRequest
* @property {BuildableComponent} component
* @property {ComponentPart} part
* @property {LoadingQuality} quality
*/
/**
* @typedef {Object} BaseLoadTracker
* @property {LoadingBase} base
* @property {Object.<UUID,Promise<null>>} loads - When the promise resolves it returns a call to this._loadNext()
* @property {number} maxConcurrentLoads
*/
/**
* Component Loader schedules and loads component sources. If no Loading Bases are given
* it creates 1 Loading Base called "autogen" with location.origin as URL.
* Scheduling works in 3 steps:
* - From the desired Buildable Components calculated an ordered list of required Loadable Components
* - Omit the Loadable Components that are already (being) loaded
* - Start at the top of the Loadables list and load
* @extends {InformationSource}
*/
class ComponentLoader extends InformationSource {
/**
* @param {Reporter} reporter
* @param {Object} settings
* @param {UUID} [settings.id]
* @param {string} [settings.name]
* @param {Array<LoadingBase>} [settings.bases]
* @param {LoadingQuality} [settings.defaultLoadingQuality='medium']
* @param {number} [settings.concurrentLoadsPerBase = 2]
*/
constructor(reporter, settings = {}) {
super(reporter, settings);
checkPropTypes(
settings,
{},
{
bases: val => Array.isArray(val) && val.every(entry => entry instanceof LoadingBase),
concurrentLoadsPerBase: 'number',
// defaultLoadingQuality: val => ['low', 'medium', 'high'].includes(val)
defaultLoadingQuality: val => Component.qualities.includes(val)
}
);
// if no loading bases are specified by the user
// create a new one here based on the location of the script
if (settings.bases) {
this.loadingBases = settings.bases;
}
else {
const origin = location.origin;
this.report({ msg: `No loading bases specified, auto generating from origin ${origin}` });
this.loadingBases = [new LoadingBase(reporter, { name: 'Autogen', url: new URL(origin) })];
}
if (settings.concurrentLoadsPerBase) {
this._concurrentLoadsPerBase = settings.concurrentLoadsPerBase;
}
// default is medium
if (settings.defaultLoadingQuality) {
this._defaultLoadingQuality = settings.defaultLoadingQuality;
}
}
/**
* Presumed number of loading sockets per browser
* @static
* @type {Object.<string,number>}
*/
static socketTable = {
'IE': 8,
'Safari': 4,
'Firefox': 6,
'Opera': 6,
'Chrome': 4
};
/**
* @type {LoadingQuality}
* @default
*/
_defaultLoadingQuality = 'medium';
set defaultLoadingQuality(quality) {
this._defaultLoadingQuality = quality;
}
/**
* Component loads can be distributed over different bases
* to prevent/minimize browser throttling
* @type {Array<LoadingBase>}
* @protected
*/
_loadingBases;
get loadingBases() {
return this._loadingBases;
}
set loadingBases(newBases) {
if (this._buildQueue.length > 0) {
throw new Error(`Can't change loading bases while loading resources`);
}
this._loadingBases = newBases;
this._currentLoads = this.loadingBases.map(loadingBase =>
({
base: loadingBase,
loads: {},
maxConcurrentLoads: this._concurrentLoadsPerBase
}));
}
/**
* Number of files that can be requested simultaneously per loading base.
* The assumption here is that the maximum number of concurrent loads
* depends on the browser, not on the host/base and therefore
* one value suffices, default is 2.
* @link http://www.websiteoptimization.com/speed/tweak/parallel/
* @link https://stackoverflow.com/questions/985431/max-parallel-http-connections-in-a-browser
* @type {number}
* @protected
* @default
*/
_concurrentLoadsPerBase = 2;
/**
* Queue of BuildableComponents in the order in which they should be loaded/built.
* By calling the .build() method, components can be added to this queue.
* @type {Array<BuildRequest>}
* @protected
*/
_buildQueue = [];
/**
* Queue of LoadableComponents in the order in which they should be loaded to
* deliver on the BuildRequests in the _buildQueue in the right order and as
* quickly as possible
* @type {Array<LoadableComponent>}
* @protected
*/
_loadableQueue = [];
/**
* Register of the current loads per loading base
* @type {Array<BaseLoadTracker>}
*/
_currentLoads = [];
/**
* Attempts to start loads on as many as possible loadable components from
* the _loadableQueue by distributing them over the loading bases in
* _currentLoads. The method is called automatically when a new buildable
* component is added to the _buildQueue[] and whenever a load is finished.
* @private
* @method
*/
_loadNext() {
// any items to load?
const nrOfLoadables = this._loadableQueue.length;
// bookkeeping the number of loads that will be initiated
let loadsAssigned = 0;
this.report({ msg: `Load next: ${nrOfLoadables > 0 ? nrOfLoadables + ' in queue.' : 'Queue empty.'}`, level: 'info' });
if (nrOfLoadables > 0) {
// iterate over the available loading bases to find an empty "slot"
for (let baseLoadTracker of this._currentLoads) {
if (this._debug) {
console.debug(baseLoadTracker.base.label, 'current loads:', Object.keys(baseLoadTracker.loads).length);
}
// Loads are "stored" in the form of a promise in the .loads property
// of the BaseLoadTracker. When the load is completed the property
// is removed from the object. So, the number of entries in the object
// is always a current indication of the number of active loads.
if (Object.keys(baseLoadTracker.loads).length < baseLoadTracker.maxConcurrentLoads) {
loadsAssigned += 1;
// Shift (remove) the "next" LoadableComponent entry from the
// queue and prepare loading it.
const nextComponentToLoad = this._loadableQueue.shift();
this.report({ msg: `Free slot in base ${baseLoadTracker.base.url.href}, start ${this._defaultLoadingQuality} quality loading ${nextComponentToLoad.label}`, level: 'debug' });
// Assign the load process an id so the Promise can later
// be deleted from the object
const loadId = uuid();
// Initiate the load by calling the .load() method on the
// loadable component. This call will (usually) reference
// the .load method in the LoadableComponent base class
// which will in turn call the ._load() method in the
// extending class.
// When the load has finished remove the entry from the
// bookkeeping object (regardless of the success or failure
// of the load) and start a new loading distribution cycle
// by calling .loadNext(). Returnec for tail call optimization.
const loadPromise = nextComponentToLoad
.load(baseLoadTracker.base, this._defaultLoadingQuality)
.finally(() => {
delete baseLoadTracker.loads[loadId];
this.report({ msg: `Load process ${nextComponentToLoad.label} finished, calling loadNext`, level: 'debug' });
return this._loadNext();
});
// Strore the promise in the tracker for bookkeeping
baseLoadTracker.loads[loadId] = loadPromise;
}
}
// If there are components to be loaded, but no empty loading
// slots, simply wait for slots to free up, but waiting for
// loading promises to fulfill
if (loadsAssigned === 0) {
// console.warn(this._currentLoads);
this.report({ msg: 'Could not assign loads to bases, all slots are taken.', level: 'debug' });
}
}
}
// buildFlag = false;
// buildReg = [];
/**
* @method
* @param {BuildableComponent} component
* @param {ComponentPart} [part = 'main']
* @param {LoadingQuality} [quality = 'medium']
* @param {Boolean} [highPriority=false]
* @returns {Promise<BuildableComponent>}
*/
async build(component, part = 'main', quality = 'medium', highPriority = false) {
//console.log( part )
// console.log('build command ', component.label, quality)
// if ( this.buildFlag === true ) {
// this.buildReg.push([component,part,quality,highPriority]);
// console.log('deferred');
// return;
// }
// this.buildFlag = true;
// /** @type {function} */
// let buildCommandProcessed = () => '';
// let buildCommandProcessedPromise = new Promise( res => buildCommandProcessed = res);
// const currentBuildCommandChain = this.buildFlag;
// currentBuildCommandChain.then( () => buildCommandProcessedPromise);
// await currentBuildCommandChain;
// console.log('accept command ', component.label)
this.report({ msg: `Build request for ${quality} quality ${part} part ${component.label}`, level: 'info' });
// if the component is already loaded simply return it
// wrapped in a promise
if (component.status[part][quality] === 'ready') {
this.report({ msg: `Component already ready, returning` });
// console.log(component)
return Promise.resolve(component);
}
// prepare a promise to return
// when all its dependencies are loaded, the component
// will update its own status, so the promise can simply
// reflect that status change
const loadCompleted = new Promise((resolve, reject) => {
const statusChangeHandler = ({ status, quality: updateQuality, part: updatePart }) => {
//console.log( part )
// console.warn(component.label, part, quality, status);
if (updateQuality === quality && updatePart === part) {
if (component._debug === true) {
console.warn(component.label, part, quality, status);
console.log(JSON.parse(JSON.stringify(component.dependencyStatus)));
}
switch (status) {
case 'loading':
case 'partially-loaded':
// return here, so the event listener isn't removed yet
return;
case 'ready':
resolve(component);
break;
case 'error':
reject(component.error[part][quality]);
break;
default:
throw new Error('Component Loader unable to handle new component status ' + status);
}
if (component.hasListener('status-change', this.id)) {
component.removeListener('status-change', this.id);
}
}
}
component.on('status-change', statusChangeHandler, this.id);
})
.finally(() => {
// remove build request
const cleanedUpBuildQueue = this._buildQueue.filter(br => br.component !== component || br.part !== part || br.quality !== quality);
const cleanupAmount = this._buildQueue.length - cleanedUpBuildQueue.length;
if (cleanupAmount > 0) {
this.report({ msg: `Cleaned up ${cleanupAmount} old build requests` });
this._buildQueue = cleanedUpBuildQueue;
}
});
/** @type {BuildRequest} */
const buildRequest = {
component,
part,
quality
};
//console.log( buildRequest )
const existingentry = this._buildQueue.find( br => br.component === component && br.part === part && br.quality === quality);
//console.log( existingentry )
if ( existingentry !== undefined ) {
// console.log('already in queue')
return loadCompleted;
}
// add the build request to the build queue
if (highPriority !== true) {
this._buildQueue.push(buildRequest);
}
else {
this._buildQueue = [buildRequest, ...this._buildQueue];
}
// console.log('build queue length', this._buildQueue.length);
// immediately build the dependencies that have no dependencies themselves
// by calling their .autoUpdateStatus() method
const allRelevantComponents = [component, ...component.allDependencies];
//console.log( 'allRelevantComponents', allRelevantComponents.length)
//console.log( allRelevantComponents )
//console.log( allRelevantComponents.map( c => c.status.main))
const depLessBCDeps = allRelevantComponents.filter(dep =>
// (dep instanceof InfoComponent)
// ||
(dep instanceof BuildableComponent && dep.dependencies && Object.values(dep.dependencies[part]).length === 0)
);
//console.log( depLessBCDeps )
const silentlyReadyDeps = allRelevantComponents.filter(dep =>
dep instanceof BuildableComponent && dep.dependencies && Object.values(dep.dependencies[part])
.every(subDep =>
Array.isArray( subDep )
? subDep.find( altSubDep => altSubDep.status[part][quality] === 'ready') // one of the alternatives
: subDep.status[part][quality] === 'ready' // the thing itself
)
);
//console.log( silentlyReadyDeps )
const synchronouslyBuildables = [...depLessBCDeps, ...silentlyReadyDeps].filter(unique);
//console.log(synchronouslyBuildables)
// console.log(synchronouslyBuildables.filter(sbb => sbb.status[part][quality] === 'not-loaded' ))
const laggingBuilds = synchronouslyBuildables
.filter(sbb => sbb.status[part][quality] === 'not-loaded' )
.map(sbb => sbb.autoUpdateStatus() );
//console.log( laggingBuilds )
// for ( let laggingBuild of laggingBuilds) {
// console.log('BUILD', laggingBuild.label)
// await laggingBuild.autoUpdateStatus();
// }
// this results in issues when multiple calls to .build are done in (close) succession
// await Promise.all(laggingBuilds);
// console.log('laggin done')
// reschedule
this._rescheduleLoadingOrder();
// return promise
// console.log('build command processed', this._loadableQueue)
// this.buildFlag = false;
// setTimeout(
// () => {
// if ( this.buildReg.length > 0 ) {
// const nextBuild = this.buildReg.splice(0,1);
// this.build
// }
// }
// )
// return Promise.all(laggingBuilds).then(() => {
// console.log('laggings')
// return loadCompleted
// });
return Promise.all(laggingBuilds).then(() => loadCompleted );
}
/**
* @private
*/
_qualitiesLowToHigh = Component.sortQualityArray(Component.qualities).reverse();
/**
* @protected
* @method
*/
_rescheduleLoadingOrder() {
// important to note here is that UI chains can not exist i.e.
// ok: main -> main -> main
// ok: UI -> main -> main
// not ok: UI -> UI -> main
/** @type {Object<LoadingQuality,Array<Component>>} */
const loadingOrderPerQuality = {};
const startQueueLength = this._loadableQueue.length;
const startBuildQueue = this._buildQueue.length;
for (let quality of Component.qualities) {
const singleQualityQueue = this._buildQueue.filter(entry =>
entry.quality === quality
);
/**
* Component list with UI dependencies inlined
* @type {Array<Component>}
*/
const compListwUIDepsInl = [];
//TO DO HIER EEN FIX GENERIEKE PATRS!!
for (let buildRequest of singleQualityQueue) {
if (buildRequest.part === 'UI') {
if ( buildRequest.component.dependencies.UI ) {
compListwUIDepsInl.push(...Object.values(buildRequest.component.dependencies.UI));
}
}
else if(buildRequest.part === 'specs') {
if ( buildRequest.component.dependencies.specs ) {
compListwUIDepsInl.push(...Object.values(buildRequest.component.dependencies.specs));
}
}
else {
compListwUIDepsInl.push(buildRequest.component)
}
}
if (compListwUIDepsInl.length > 0) {
loadingOrderPerQuality[quality] = ComponentLoader.optimalLoadingOrder(
compListwUIDepsInl,
quality
);
}
}
this._loadableQueue = this._qualitiesLowToHigh.map(quality =>
loadingOrderPerQuality[quality]?.relevantDependencies || []
).flat();
const endQueueLength = this._loadableQueue.length;
const endBuildQueue = this._buildQueue.length;
this.report({ msg: `Rescheduled loading order, ${startQueueLength} => ${endQueueLength}, ${startBuildQueue} - ${endBuildQueue}` })
this._loadNext();
}
/**
* Determines the optimal loading order for an array of components that
* need to be loaded at the same quality, only taking into account the 'main'
* part of the component.
* @static
* @param {Array<Component>} components
* @param {LoadingQuality} [quality='medium']
*/
static optimalLoadingOrder(components, quality = 'medium') {
// make a list of lists of needed loadable components
const dependencySets = ComponentLoader.getLoadableDependencies(components);
// console.log( 'dependencySets', dependencySets)
// dependencies that are loaded or currently loading should be ignored
const relevantStates = ['not-loaded', 'partially-loaded'];
// return the one with the smallest expected size
const sizedDepSets = dependencySets
.map(depSet => {
// Set to array
const dependencies = [...depSet];
// filter relevant deps
const relevantDependencies = dependencies.filter(dep => relevantStates.includes(dep.status.main[quality]));
// console.log( relevantDependencies)
// console.log( relevantDependencies.forEach(dep => console.log( dep.label, dep.status.main[quality])))
return {
dependencies,
relevantDependencies,
sizeBytes: relevantDependencies
.reduce(
(size, dep) => {
if (relevantStates.includes(dep.status.main[quality])) {
return size + dep.source[quality].sizeBytes;
}
else {
// dependency is already loaded or currently loading
return size;
}
},
0
)
}
});
const smallestDepSet = sizedDepSets.sort((a, b) => a.sizeBytes - b.sizeBytes)[0];
// console.log(sizedDepSets, 'smallest', smallestDepSet);
return smallestDepSet;
// components subscribe to load events from their dependencies and set themselves "usable" when
// all dependencies are loaded, so the responsibility for state control is still
// within the component
// however, as a component can't easily look "up" in the tree, it won't know
// which loading alternative to pick if there are multiple loading paths
}
/**
* Builds sets of dependency paths, only taking into account the main part of components
* @static
* @param {(Component|Array<Component>)} stuffToLoad
* @param {Array<Set>} [depSetsArr]
*/
static getLoadableDependencies(stuffToLoad, /* out */ depSetsArr = [new Set()], /* out */ analysedComponents = []) {
// console.log( 'getLoadableDependencies', stuffToLoad.constructor.name, stuffToLoad instanceof LoadableComponent, stuffToLoad instanceof BuildableComponent)
if (stuffToLoad instanceof LoadableComponent) {
// console.log( 'Adding loadable:', stuffToLoad.label)
depSetsArr.forEach(set => set.add(stuffToLoad));
//console.log(depSetsArr)
}
else {
let componentList = null;
if (stuffToLoad instanceof BuildableComponent) {
componentList = Object.values(stuffToLoad.dependencies.main);
// console.log( 'BC', componentList)
}
else if (Array.isArray(stuffToLoad) && stuffToLoad.every(entry => entry instanceof Component)) {
componentList = stuffToLoad;
// console.log( 'ARR', componentList)
}
else if (stuffToLoad instanceof InfoComponent) {
componentList = [];
// console.log( 'Info', componentList)
}
else {
// console.warn(stuffToLoad);
throw new Error('Cannot get dependencies of ' + typeof (stuffToLoad));
}
for (let component of componentList) {
if (component instanceof Component) {
if (analysedComponents.indexOf(component) === -1) {
// console.log( 'Analysing', stuffToLoad.label)
// components in the analysedComponents list will not be analysed again
// therefore, only non-alternative
analysedComponents.push(stuffToLoad);
depSetsArr = this.getLoadableDependencies(component, depSetsArr, analysedComponents);
}
else {
// console.log(`Already analysed ${component.label} - skipping`);
// console.log(analysedComponents, analysedComponents.indexOf(component))
continue;
}
}
else if (Array.isArray(component)) {
const newDepSetsArr = [];
// console.log( 'Array alternatives', component.length)
for (let subDep of component) {
if (analysedComponents.indexOf(subDep) === -1) {
const depSetsArrClone = depSetsArr.map(depSet => new Set(depSet));
newDepSetsArr.push(...this.getLoadableDependencies(subDep, depSetsArrClone, analysedComponents.slice()));
}
else {
// console.log(`Already analysed ${subDep.label} - skipping`);
continue;
}
}
if (newDepSetsArr.length > 0) {
depSetsArr = newDepSetsArr;
}
}
}
}
return depSetsArr;
}
// /**
// * @static
// * @param {LoadableComponent} component
// */
// static getLoader(component) {
// switch (component.constructor.name) {
// case 'WrappedImage':
// return ComponentLoader.imageLoader;
// case 'GeometryFile':
// return ComponentLoader.GLTFLoader;
// case 'JSONFile':
// return ComponentLoader.fileLoader;
// default:
// throw new Error('Unknown component type');
// }
// }
}
export { ComponentLoader };