contract.txt

Create Feed:
    SADD feeds [feed] //if fail, abort
    PUBLISH newfeed [feed]\x00[instance uuid]

Delete Feed:
    WATCH feeds
    SISMEMBER feeds [feed] //if nil: fail
    MULTI
        SREM feeds [feed]
        for key associated with feed:
            DELETE [key]
        PUBLISH delfeed [feed]\x00[instance uuid]
    EXEC // if nil: go back to WATCH

Set Config Value:
    HSET feed.config:[feed] name value
    PUBLISH conffeed [feed]\x00[instance uuid]

Get Config Value:
    HGET feed.config:[feed] name

Feed:

    Publish:
        //id may be provided or generated. An id that has already been published will update that id
    
        max = feed.config:[feed] max_length
        WATCH feed.ids:[feed]
        delete_ids = ZRANGE feed.ids:[feed] 0 [-max] // eg ZRANGE feed.ids:test 0 -5 if the max is 4
        MULTI
            for id in delete_ids:
                if id != publish_id:
                    ZREM feed.ids:[feed] [id]
                    HDEL feed.items:[feed] [id]
                    PUBLISH feed.retract:[feed] [id]
            ZADD feed.ids:[feed] [utc epoch milliseconds] [id]
            INCR feed.publishes:[feed]
            HSET feed.items:[feed] [id] [item]
        EXEC // if nil, go back to WATCH
        //if zadd returned 1
            PUBLISH feed.publish:[feed] [id]\x00[item]
        //else
            PUBLISH feed.edit:[feed] [id]\x00[item]


    Retract:
        WATCH feed.ids:[feed]
        if ZRANK feed.ids:[feed] [id]
            MULTI
                ZREM feed.ids:[feed] [id]
                HDEL feed.items:[feed] [id]
                PUBLISH feed.retract:[feed] [id]
            EXEC // if nil, go back to WATCH

    Get IDs:
        ZRANGE feed.ids:[feed] 0 -1

    Get Items:
        HGET feed.items:[feed] [id]

    Get All:
        HGETALL feed.items:[feed]


Sorted Feed:

    Append/Publish:
        id = INCR feed.idincr:[feed]
        MULTI
            RPUSH feed.ids:[feed] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
            PUBLISH feed.publish:[feed] [id]\x00[item]
            PUBLISH feed.position:[feed] [id]\x00:end
        EXEC

    Prepend:
        id = INCR feed.idincr:[feed]
        MULTI
            LPUSH feed.ids:[feed] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
            PUBLISH feed.publish:[feed] [id]\x00[item]
            PUBLISH feed.position:[feed] [id]\x00begin:
        EXEC

    Edit:
        WATCH feed.items:[feed]
        HEXISTS feed.items:[feed] [id] //if not: UNWATCH fail
        MULTI
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
            PUBLISH feed.publish:[feed] [id]\x00[item]
        EXEC //if nil, retry at WATCH

    Publish Before:
        WATCH feed.items:[feed]
        HEXISTS feed.items:[feed] [before id] //if not: UNWATCH fail
        id = INCR feed.idincr:[feed]
        MULTI
            LINSERT feed.ids:[feed] BEFORE [before id] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
            PUBLISH feed.publish:[feed] [id]\x00[item]
            PUBLISH feed.position:[feed] [id]\x00:[before id]
        EXEC // if nil: retry at WATCH
    
    Publish After:
        WATCH feed.items:[feed]
        HEXISTS feed.items:[feed] [after id] //if not: UNWATCH, fail
        id = INCR feed.idincr:[feed]
        MULTI
            LINSERT feed.ids:[feed] AFTER [after id] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
            PUBLISH feed.publish:[feed] [id]\x00[item]
            PUBLISH feed.position:[feed] [id]\x00[after id]:
        EXEC // if nil: retry at WATCH

    Move:
        //relative_position = :3 for before 3, 3: for after 3
        // = :end for end, begin: for begin
        WATCH feed.items:[feed]
        HEXISTS feed.items:[feed] [id] //if not: UNWATCH, fail
        //if relative_position is not :end or begin:
            HEXISTS feed.items:[feed] [relative id] //if not: UNWATCH, fail
        MULTI
            LREM feed.ids:[feed] 1 [id]
            //if begin:
                LPUSH feed.ids:[feed] [id]
            //if :end
                RPUSH feed.ids:[feed] [id]
            //else
                LINSERT feed.ids:[feed] [BEFORE/AFTER] [relative id] [id]
            PUBLISH feed.position:[feed] [id]\x00[relative position]
        EXEC // if nil: retry at WATCH
    
    Retract:
        WATCH feed.items:[feed]
        if HEXISTS feed.items:[feed] [id]
            MULTI
                LREM feed.ids:[feed] 1 [id]
                HDEL feed.items:[feed] [id]
                PUBLISH feed.retract:[feed] [id]
            EXEC // if nil, go back to WATCH
        else UNWATCH feed.ids:[feed]

    Get IDs:
        LRANGE feed.ids:[feed] 0 -1

    Get Items:
        HGET feed.items:[feed] [id]

    Get All:
        HGETALL feed.items:[feed]


Queue:

    Normal Priority Put:
        // id = generated uuid
        MULTI
            LPUSH feed.ids:[feed] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
        EXEC

    High Priority Put:
        // id = generated uuid
        MULTI
            RPUSH feed.ids:[feed] [id]
            HSET feed.items:[feed] [id] [item]
            INCR feed.publishes:[feed]
        EXEC

    Get:
        id = BRPOP feed.ids:[feed] [timeout]
        //if error, abort
        MULTI
            item = HGET feed.items:[feed] [id]
            HDEL feed.items:feed [id]
        EXEC

    Get IDs:
        LRANGE feed.ids:[feed] 0 -1

Job:

    Normal Priority Put:
        // id = generated uuid
        MULTI
            LPUSH feed.ids:[feed] [id]
            INCR feed.publishes:[feed]
            HSET feed.items:[feed] [id] [item]
            ZADD feed.published:[feed] [utc epoch milliseconds] [id]
        EXEC

    High Priority Put:
        // id = generated uuid
        MULTI
            RPUSH feed.ids:[feed] [id]
            HSET feed.items:[feed] [id] [item]
            ZADD feed.published:[feed] [utc epoch milliseconds] [id]
        EXEC

    Get:
        id = BRPOP feed.ids:[feed] [timeout]
        //if error/timeout, abort
        MULTI
            ZADD feed.claimed:[feed] [utc epoch milliseconds] [id]
            item = HGET feed:items[feed] [id]
        EXEC
        //if the id fails to get from feed.ids to feed.claimed, the maintenance will notice eventually

    Finish: //worker finishes job
        WATCH feed.claimed:[feed]
        ZRANK feed.claimed:[feed] [id] // if nil: fail, this job isn't claimed and you can't finish it
        MULTI
            ZREM feed.claimed:[feed] [id]
            HDEL feed.cancelled:[feed] [id] //just to make sure
            INCR feed.finishes:[feed]
            //optionally if publishing a result:
                PUBLISH job.finish:[feed] [id]\x00[result]
            HDEL feed.items:[feed] [id]
        EXEC // if nil: go back to WATCH and try again

    Get Ids:
        HKEYS feed.items:[feed]

    Cancel: //cancel a running job, will get retried
        WATCH feed.claimed:[feed]
        ZRANK feed.claimed:[feed] [id] // if nil: fail UNWATCH
        MULTI
            HINCRBY feed.cancelled:[feed] [id] 1
            LPUSH feed.ids:[feed] [id]
            ZREM feed.claimed:[feed] [id]
        EXEC // if nil: go back to watch and try again

    Stall: //fail a claimed job semi-permantently and take it out of the running, but don't delete
        WATCH feed.claimed:[feed]
        ZRANK feed.claimed:[feed] [id] // if nil: fail
        MULTI
            ZREM feed.claimed:[feed] [id]
            HDEL feed.cancelled:[feed] [id]
            SADD feed.stalled:[feed] [id]
            ZREM feed.published:[feed] [id]
        EXEC // if nil, retry

    Retry: //retry a stalled job
        WATCH feed.stalled:[feed]
        SISMEMBER feed.stalled:[feed] [id] // if nil: UNWATCH fail
        MULTI
            SREM feed.stalled:[feed] [id]
            //if error, abort
            LPUSH feed.ids:[feed] [id]
            ZADD feed.published:[feed] [utc epoch milliseconds] [id]
        EXEC // if nil retry

    Retract:
        WATCH feed.items:[feed]
        if not HEXISTS feed.items:[feed] [id] // fail UNWATCH
        MULTI
            HDEL feed.items:[feed] [id]
            HDEL feed.cancelled:[feed] [id]
            ZREM feed.published:[feed] [id]
            SREM feed.stalled:[feed] [id]
            ZREM feed.claimed:[feed] [id]
            LREM feed.ids:[feed] 1 [id]
        EXEC // if fail, retry

    getNumOfFailures:
        HGET feed.cancelled:[feed] [id]

    Maintenance: //maintain job queue -- only ran by one process per jobqueue on occassion -- still a bit hand-wavey
        MULTI
            keys = HKEYS feed.items:[feed]
            avail = LRANGE feed.ids:[feed]
            claim = ZRANGE feed.claimed:[feed] 0 -1
            stall = SMEMBERS feed.stalled:[feed]
        EXEC
        unaccounted = any of keys not in avail, claim, stall
        for key in unnaccounted_last:
            if key in keys and not in avail, claim, stall:
                LPUSH feed.ids:[feed] [key]

        check claimed jobs to see if any have been claimed too long and "Cancel" or "Stall" them
        publish stats to a feed