Where does MM5 store track waveforms? [#17038]

[drakinite]

I'm curious to see how MM5 computes and stores the track waveforms; Where are they stored? Are they stored somewhere in MM5.db as some sort of sequence of numbers, or are they stored as images somewhere?

[MiPi]

They are currently cached in Portable\Temp\Waveforms (for portable installation) as binary files with min and max values (current resolution is 4096 values). Values are taken from decoded 16bit Mono PCM.

[drakinite]

Whoa, cool! I can even import them into Audacity as Raw Data. They sound pretty funky when played as audio.

How come MM doesn't normalize some of the quieter tracks, like these two? https://i.imgur.com/HygGmQv.png

[MiPi]

Did you calculate normalization in MM - Tools - Analyze volume? Waveform computation uses normalization coefficient saved to the file (or DB), it does not do any normalization on its own. Btw. you need to rightclick waveform and select "Update waveform" to recompute it then.

[drakinite]

Hmm. It looks like for loud tracks, which have a negative value for analyzed track volume, the resulting waveform becomes very thin. But for quiet tracks, which have a positive value for analyzed track volume, the waveform looks good.

Perhaps the waveform analyzer should only accept positive normalization coefficients, and treat negative normalization coefficients as zero? (Let me know if you want me to send pictures to show what I mean.)

[MiPi]

Could you send me such affected file with loud track? So I can try it and do experiments with the same data. You can use any file sharing service and PM me link. Generally too loud tracks are problematic, because they have very bad dynamic, the waveform is not very nice then, it is simply rectangle of loudness. And normalizing only makes its height smaller, but it will be still not very nice rectangle.

[MiPi]

Fixed as https://www.ventismedia.com/mantis/view.php?id=17038

[drakinite]

Nice! Thanks MiPi!

[drakinite]

I tweaked the waveform.js code to add a cache of waveform data, which causes them to load faster
Sometimes the waveforms still take a while to load, especially when you spam the next/prev track buttons, but I think it's caused by other parts of MM, and not the caching code. This code preloads the next track's waveform, which (usually) causes it to load instantly when you hit the next track button

Code: Select all

/* '(C) Ventis Media, Licensed under the Ventis Limited Reciprocal License - see: license.txt for details' */

"use strict";

/**
@module UI
*/

requirejs('controls/slider');


/**
UI Waveform element

@class Waveform
@constructor
@extends Control
*/

var cache = {
    lastTrack: {sd: undefined, values: undefined},
    currentTrack: {sd: undefined, values: undefined},
    nextTrack: {sd: undefined, values: undefined},
}

inheritClass('Waveform', Slider, {
    initialize: function (parentel, params) {
        params = params || {};
        params.orientation = 'horizontal';
        params.invert = false;
        params.tickPlacement = 'none';
        params.mergeParentContextMenu = true;
        Waveform.$super.initialize.call(this, parentel, params);

        var _this = this;
        this.container.classList.add('no-cpu');
        this.container.classList.add('waveform');
        this._width = 0;
        this._height = 0;
        this._visCanvas = document.createElement('canvas');
        this._seekBar.appendChild(this._visCanvas);
        this._wasVisible = this.visible;
        this._currState = undefined;
        this._visCanvasCtx = null;
        this._fillStyle1 = getStyleRuleValue('color', '.textcolor') || 'white';

        this.handle_layoutchange();

        var player = app.player;

        // start visualization when playback is already running (e.g. skin was changed)
        if (player.paused)
            this.onPlaybackState('pause');
        else if (player.isPlaying)
            this.onPlaybackState('play');
        this.registerEventHandler('layoutchange');
        if (this.animateProgress)
            this.toggleAnimate(true);

        this.localListen(thisWindow, 'closequery', function (token) {
            if (this._getWFPromise) {
                token.asyncResult = true;
                this.waveCalcCancelled = false;
                //ODS('--- cancelling getWave');
                this.cancelGetWaveform();

                var _checkTerminated = () => {
                    if (!this.waveCalcCancelled) {
                        requestTimeout(() => {
                            _checkTerminated();
                        }, 10);
                    } else {
                        //ODS('--- cancelled getWave');
                        token.resolved();
                    }
                }

                _checkTerminated();
            } else {
                token.resolved();
            }
        }.bind(this));

        this.addToContextMenu(function () {
            return [{
                action: {
                    title: _('Update waveform'),
                    execute: function () {
                        _this.cancelGetWaveform();
                        _this.clearWave();
                        _this.values = undefined;
                        _this.updateWaveform(true);
                    },
                    visible: function () {
                        return !_this._getWFPromise;
                    }
                },
                order: 10,
                grouporder: 0
            }];
        });
    },

    updateWaveform: function (forceUpdate) {
        var _this = this;
        if (_this.visible && !_this.values && !_this._getWFPromise && _this.songLength && !_this.isVideo && _this.sd.path != '') {
            ODS('WF ' + _this.uniqueID + ' ' + _this.container.getAttribute('data-id') + ' ' + document._isEmbedded + ': getting waveform for ' + _this.sd.path);
            var calledForSD = _this.sd;
            
            var id = calledForSD.idsong;
            if (id) console.log(`Song id = ${id}; forceUpdate = ${forceUpdate}`);
            if (cache.lastTrack.sd && calledForSD.isSame(cache.lastTrack.sd) && !forceUpdate) {
                // if lastTrack matches, then swap currentTrack for lastTrack
                var lastSD = cache.currentTrack.sd;
                var lastValues = cache.currentTrack.values;
                cache.currentTrack.sd = cache.lastTrack.sd;
                cache.currentTrack.values = cache.lastTrack.values;
                cache.lastTrack.sd = lastSD;
                cache.lastTrack.values = lastValues;
                
                ODS('WF ' + _this.uniqueID + ': got waveform from last-track cache, calling redraw for ' + _this.sd.path);
                _this.values = cache.currentTrack.values;
                _this.redraw();
            }
            else if (cache.currentTrack.sd && calledForSD.isSame(cache.currentTrack.sd) && !forceUpdate) {
                ODS('WF ' + _this.uniqueID + ': got waveform from current-track cache, calling redraw for ' + _this.sd.path);
                _this.values = cache.currentTrack.values;
                _this.redraw();
            }
            else if (cache.nextTrack.sd && calledForSD.isSame(cache.nextTrack.sd) && !forceUpdate) {
                // put currentTrack into lastTrack & nextTrack into currentTrack
                cache.lastTrack.sd = cache.currentTrack.sd;
                cache.lastTrack.values = cache.currentTrack.values;
                cache.currentTrack.sd = cache.nextTrack.sd;
                cache.currentTrack.values = cache.nextTrack.values;
                
                ODS('WF ' + _this.uniqueID + ': got waveform from next-track cache, calling redraw for ' + _this.sd.path);
                _this.values = cache.currentTrack.values;
                _this.redraw();
            }
            // if wf could not be found in cache
            else {
                _this.getWaveformData(calledForSD, forceUpdate, (err, values) => {
                    if (err) {
                        //log error
                        ODS(err);
                        calledForSD = undefined;
                        //end
                        return;
                    }
                    if (values) {
                        _this.values = values;
                        calledForSD = undefined;
                        ODS('WF ' + _this.uniqueID + ': values set, calling redraw for ' + _this.sd.path);
                        _this.redraw();
                        
                        // put current track cache values into last track cache
                        if (cache.currentTrack.sd && cache.currentTrack.values) {
                            cache.lastTrack.sd = cache.currentTrack.sd;
                            cache.lastTrack.values = cache.currentTrack.values;
                        }
                        
                        // set current track cache values
                        cache.currentTrack.sd = calledForSD;
                        cache.currentTrack.values = values;
                        
                        ODS('WF: getting values for upcoming track');
                        
                        // now, get next track ahead-of-time
                        try {
                            var nextTrackSD;
                            nextTrackSD = player.getFastNextTrack(nextTrackSD);
                            if (nextTrackSD) {
                                _this.getWaveformData(nextTrackSD, forceUpdate, (err, values) => {
                                    if (err) {
                                        //log error
                                        ODS(err);
                                        nextTrackSD = undefined;
                                        //end
                                        return;
                                    }
                                    if (values) {
                                        ODS('WF ' + _this.uniqueID + ': values set for ' + nextTrackSD.path + ', saved to cache');
                                        // put values and sd into next track cache
                                        cache.nextTrack.sd = nextTrackSD;
                                        cache.nextTrack.values = values;
                                    }
                                })
                            }
                        }
                        catch(err) {
                            ODS('WF: Could not get next track info.')
                        }
                    }
                })
            }
            
        };
    },
    
    getWaveformData: function(sd, forceUpdate, cb) {
        var _this = this;
        
        _this._getWFPromise = app.player.getWaveform(sd, !!forceUpdate);
        _this._getWFPromise.then(
            // on complete
            function (ret) {
                if (!ret) {
                    if (sd.isSame(_this.sd)) {
                        _this._getWFPromise = undefined;
                        _this.waveCalcCancelled = true;
                    }
                    //callback with error
                    return cb('WF ' + _this.uniqueID + ' no waveform for ' + sd.path);
                }
                _this._getWFPromise = undefined;
                var values = JSON.parse(ret);
                cb(null, values);
            }, 
            // on reject
            function () {
                if(_this.sd) {
                    if (sd.isSame(_this.sd)) {
                        _this.waveCalcCancelled = true;
                        _this._getWFPromise = undefined;
                    }
                };
                cb('WF ' + _this.uniqueID + ' refused waveform for ' + sd.path);
            }
        );
    },

    onPlaybackState: function (state, sd) {
        var _this = this;
        switch (state) {
            case 'play':
            case 'unpause':
            case 'pause':
                this.disabled = window.settings.UI.disablePlayerControls['seek'];
                this._currState = state;
                this.updateWaveform();
                break;
            case 'end':
            case 'stop':
                this.disabled = true;
                this._currState = state;
                break;
            case 'trackChanged':
                this.dataSource = sd;
                if ((this._currState === 'play') || (this._currState === 'pause') || (this._currState === 'unpause')) {
                    if (sd && (sd.path === '')) {
                        this.dataSourceListen(sd, 'change', () => {
                            if (sd.path !== '')
                                this.requestTimeout(() => {
                                    this.updateWaveform();
                                }, 500, '_updateWaveFormTm');
                        });
                    } else
                        this.updateWaveform();
                }
                break;
        }
    },

    toggleAnimate: function (anim) {
        if (!this.animateProgress)
            return;
        if (this.lastAnimate !== anim) {
            this.lastAnimate = anim;
            this._seekBarThumb.classList.toggle('animate', anim);
            this._seekBarBefore.classList.toggle('animate', anim);
        }
    },

    redraw: function (changedVisibility) {
        if (!this.sd || !this._wasVisible || this.isVideo || !this._visCanvasCtx || !this._visCanvas.width || !this._visCanvas.height)
            return;
        var x = 0;
        var yMin = 0;
        var yMax = 0;
        this.clearWave();
        if (this.values) {
            //ODS('--- WF ' + this.uniqueID + ': drawing to (' + this._visCanvas.width + ', ' + this._visCanvas.height + '), ' + this.sd.path);
            this._visCanvasCtx.beginPath();
            this._visCanvasCtx.lineWidth = '1';
            this._visCanvasCtx.strokeStyle = this._fillStyle1;
            var stepX = this._visCanvas.width / this.values.length;
            var stepY = this._visCanvas.height / 65535;
            for (var i = 0; i < this.values.length; i++) {
                yMin = this._visCanvas.height - stepY * (this.values[i][0] + 32768);
                yMax = this._visCanvas.height - stepY * (this.values[i][1] + 32768);
                this._visCanvasCtx.moveTo(x, yMin);
                this._visCanvasCtx.lineTo(x, yMax);
                x += stepX;
            }
            this._visCanvasCtx.stroke();
            this._clearedCanvas = false;
        } else if (changedVisibility) {
            //ODS('--- WF ' + this.uniqueID + ': redraw - called ' + this._currState + ' for ' + this.sd.path);
            this.onPlaybackState(this._currState);
        } // else
        //ODS('--- WF ' + this.uniqueID +': redraw - do nothing for ' + this.sd.path);
    },

    handle_layoutchange: function (evt) {
        var wasVis = this._wasVisible;
        this._wasVisible = this.visible;
        if (this._wasVisible && this._visCanvas) {
            var cs = window.getComputedStyle(this._seekBar, null);
            var w = Math.round(Math.abs(parseFloat(cs.getPropertyValue('width'))));
            var h = Math.round(Math.abs(parseFloat(cs.getPropertyValue('height'))));

            this._height = h;
            this._width = w;

            if (!wasVis || (this._visCanvas.width !== w) || (this._visCanvas.height !== h)) {
                this._visCanvas.width = w;
                this._visCanvas.height = h;

                this._visCanvasCtx = this._visCanvas.getContext('2d');
                if (!this._getWFPromise)
                    this.redraw(this._wasVisible !== wasVis);
            } else if (!this.values)
                this.redraw(this._wasVisible !== wasVis);
        } else {
            this.cancelGetWaveform();
        }
        if (evt)
            Waveform.$super.handle_layoutchange.call(this, evt);
    },

    clearWave: function () {
        if (!this._clearedCanvas && this._visCanvasCtx) {
            this._visCanvasCtx.clearRect(0, 0, this._visCanvas.width, this._visCanvas.height);
            this._clearedCanvas = true;
        }
    },

    cancelGetWaveform: function () {
        if (this._getWFPromise) {
            cancelPromise(this._getWFPromise);
            this._getWFPromise = undefined;
        }
    },

    cleanUp: function () {
        this.cancelGetWaveform();
        Waveform.$super.cleanUp.call(this);
    }
}, {
    dataSource: {
        get: function () {
            return this.sd;
        },
        set: function (sd) {
            if ((!this.sd && !sd) || (this.sd && sd && this.sd.isSame(sd)))
                return;

            this.dataSourceUnlistenFuncts();
            this.sd = sd;
            this.cancelGetWaveform();
            this.clearWave();
            this.values = undefined;
            if (!this.sd) {
                this.songLength = 0;
                this.max = 0;
                return;
            }
            this.isVideo = this.sd.isVideo || _utils.isYoutubeTrack(this.sd);
            this.songLength = this.sd.songLength;
            this.max = this.songLength / 1000.0;
            //ODS('--- WF ' + this.uniqueID + ': dataSource= ' + this.sd.path);
        }
    },
});

[MiPi]

Good idea, we were postponing this till now We can include it in the default code I think, with some tweaks around getFastNextTrack/getNextTrack, as mentioned in different thread. It will not work in shuffle mode, where decision about next track is made just before going to the next track, but could be handy otherwise. Thanks!

[drakinite]

Nice - Glad you like it!