nav tabs on admin dashboard

This commit is contained in:
2019-03-07 00:20:34 -06:00
parent f73d6ae228
commit e4f473f376
11661 changed files with 216240 additions and 1544253 deletions
+112 -42
View File
@@ -3,21 +3,25 @@
const BB = require('bluebird')
const contentPath = require('./path')
const figgyPudding = require('figgy-pudding')
const fs = require('graceful-fs')
const PassThrough = require('stream').PassThrough
const pipe = BB.promisify(require('mississippi').pipe)
const ssri = require('ssri')
const Y = require('../util/y.js')
BB.promisifyAll(fs)
const lstatAsync = BB.promisify(fs.lstat)
const readFileAsync = BB.promisify(fs.readFile)
const ReadOpts = figgyPudding({
size: {}
})
module.exports = read
function read (cache, integrity, opts) {
opts = opts || {}
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
return fs.readFileAsync(cpath, null).then(data => {
opts = ReadOpts(opts)
return withContentSri(cache, integrity, (cpath, sri) => {
return readFileAsync(cpath, null).then(data => {
if (typeof opts.size === 'number' && opts.size !== data.length) {
throw sizeError(opts.size, data.length)
} else if (ssri.checkData(data, sri)) {
@@ -29,17 +33,31 @@ function read (cache, integrity, opts) {
})
}
module.exports.sync = readSync
function readSync (cache, integrity, opts) {
opts = ReadOpts(opts)
return withContentSriSync(cache, integrity, (cpath, sri) => {
const data = fs.readFileSync(cpath)
if (typeof opts.size === 'number' && opts.size !== data.length) {
throw sizeError(opts.size, data.length)
} else if (ssri.checkData(data, sri)) {
return data
} else {
throw integrityError(sri, cpath)
}
})
}
module.exports.stream = readStream
module.exports.readStream = readStream
function readStream (cache, integrity, opts) {
opts = opts || {}
opts = ReadOpts(opts)
const stream = new PassThrough()
pickContentSri(
cache, integrity
).then(content => {
const sri = content.sri
withContentSri(cache, integrity, (cpath, sri) => {
return lstatAsync(cpath).then(stat => ({cpath, sri, stat}))
}).then(({cpath, sri, stat}) => {
return pipe(
fs.createReadStream(contentPath(cache, sri)),
fs.createReadStream(cpath),
ssri.integrityStream({
integrity: sri,
size: opts.size
@@ -52,37 +70,93 @@ function readStream (cache, integrity, opts) {
return stream
}
let copyFileAsync
if (fs.copyFile) {
module.exports.copy = copy
module.exports.copy.sync = copySync
copyFileAsync = BB.promisify(fs.copyFile)
}
function copy (cache, integrity, dest, opts) {
opts = opts || {}
return pickContentSri(cache, integrity).then(content => {
const sri = content.sri
const cpath = contentPath(cache, sri)
return fs.copyFileAsync(cpath, dest).then(() => content.size)
opts = ReadOpts(opts)
return withContentSri(cache, integrity, (cpath, sri) => {
return copyFileAsync(cpath, dest)
})
}
function copySync (cache, integrity, dest, opts) {
opts = ReadOpts(opts)
return withContentSriSync(cache, integrity, (cpath, sri) => {
return fs.copyFileSync(cpath, dest)
})
}
module.exports.hasContent = hasContent
function hasContent (cache, integrity) {
if (!integrity) { return BB.resolve(false) }
return pickContentSri(cache, integrity)
.catch({code: 'ENOENT'}, () => false)
.catch({code: 'EPERM'}, err => {
if (process.platform !== 'win32') {
throw err
} else {
return false
return withContentSri(cache, integrity, (cpath, sri) => {
return lstatAsync(cpath).then(stat => ({size: stat.size, sri, stat}))
}).catch(err => {
if (err.code === 'ENOENT') { return false }
if (err.code === 'EPERM') {
if (process.platform !== 'win32') {
throw err
} else {
return false
}
}
}).then(content => {
if (!content.sri) return false
return ({ sri: content.sri, size: content.stat.size })
})
}
module.exports._pickContentSri = pickContentSri
function pickContentSri (cache, integrity) {
module.exports.hasContent.sync = hasContentSync
function hasContentSync (cache, integrity) {
if (!integrity) { return false }
return withContentSriSync(cache, integrity, (cpath, sri) => {
try {
const stat = fs.lstatSync(cpath)
return {size: stat.size, sri, stat}
} catch (err) {
if (err.code === 'ENOENT') { return false }
if (err.code === 'EPERM') {
if (process.platform !== 'win32') {
throw err
} else {
return false
}
}
}
})
}
function withContentSri (cache, integrity, fn) {
return BB.try(() => {
const sri = ssri.parse(integrity)
// If `integrity` has multiple entries, pick the first digest
// with available local data.
const algo = sri.pickAlgorithm()
const digests = sri[algo]
if (digests.length <= 1) {
const cpath = contentPath(cache, digests[0])
return fn(cpath, digests[0])
} else {
return BB.any(sri[sri.pickAlgorithm()].map(meta => {
return withContentSri(cache, meta, fn)
}, {concurrency: 1}))
.catch(err => {
if ([].some.call(err, e => e.code === 'ENOENT')) {
throw Object.assign(
new Error('No matching content found for ' + sri.toString()),
{code: 'ENOENT'}
)
} else {
throw err[0]
}
})
}
})
}
function withContentSriSync (cache, integrity, fn) {
const sri = ssri.parse(integrity)
// If `integrity` has multiple entries, pick the first digest
// with available local data.
@@ -90,21 +164,17 @@ function pickContentSri (cache, integrity) {
const digests = sri[algo]
if (digests.length <= 1) {
const cpath = contentPath(cache, digests[0])
return fs.lstatAsync(cpath).then(stat => ({ sri: digests[0], stat }))
return fn(cpath, digests[0])
} else {
return BB.any(sri[sri.pickAlgorithm()].map(meta => {
return pickContentSri(cache, meta)
}))
.catch(err => {
if ([].some.call(err, e => e.code === 'ENOENT')) {
throw Object.assign(
new Error('No matching content found for ' + sri.toString()),
{code: 'ENOENT'}
)
} else {
throw err[0]
let lastErr = null
for (const meta of sri[sri.pickAlgorithm()]) {
try {
return withContentSriSync(cache, meta, fn)
} catch (err) {
lastErr = err
}
})
}
if (lastErr) { throw lastErr }
}
}
+3 -1
View File
@@ -28,7 +28,9 @@ function write (cache, data, opts) {
if (typeof opts.size === 'number' && data.length !== opts.size) {
return BB.reject(sizeError(opts.size, data.length))
}
const sri = ssri.fromData(data, opts)
const sri = ssri.fromData(data, {
algorithms: opts.algorithms
})
if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) {
return BB.reject(checksumError(opts.integrity, sri))
}
+96 -31
View File
@@ -4,6 +4,7 @@ const BB = require('bluebird')
const contentPath = require('./content/path')
const crypto = require('crypto')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
const hashToSegments = require('./util/hash-to-segments')
@@ -29,9 +30,16 @@ module.exports.NotFoundError = class NotFoundError extends Error {
}
}
const IndexOpts = figgyPudding({
metadata: {},
size: {},
uid: {},
gid: {}
})
module.exports.insert = insert
function insert (cache, key, integrity, opts) {
opts = opts || {}
opts = IndexOpts(opts)
const bucket = bucketPath(cache, key)
const entry = {
key,
@@ -67,10 +75,36 @@ function insert (cache, key, integrity, opts) {
})
}
module.exports.insert.sync = insertSync
function insertSync (cache, key, integrity, opts) {
opts = IndexOpts(opts)
const bucket = bucketPath(cache, key)
const entry = {
key,
integrity: integrity && ssri.stringify(integrity),
time: Date.now(),
size: opts.size,
metadata: opts.metadata
}
fixOwner.mkdirfix.sync(path.dirname(bucket), opts.uid, opts.gid)
const stringified = JSON.stringify(entry)
fs.appendFileSync(
bucket, `\n${hashEntry(stringified)}\t${stringified}`
)
try {
fixOwner.chownr.sync(bucket, opts.uid, opts.gid)
} catch (err) {
if (err.code !== 'ENOENT') {
throw err
}
}
return formatEntry(cache, entry)
}
module.exports.find = find
function find (cache, key) {
const bucket = bucketPath(cache, key)
return bucketEntries(cache, bucket).then(entries => {
return bucketEntries(bucket).then(entries => {
return entries.reduce((latest, next) => {
if (next && next.key === key) {
return formatEntry(cache, next)
@@ -87,11 +121,36 @@ function find (cache, key) {
})
}
module.exports.find.sync = findSync
function findSync (cache, key) {
const bucket = bucketPath(cache, key)
try {
return bucketEntriesSync(bucket).reduce((latest, next) => {
if (next && next.key === key) {
return formatEntry(cache, next)
} else {
return latest
}
}, null)
} catch (err) {
if (err.code === 'ENOENT') {
return null
} else {
throw err
}
}
}
module.exports.delete = del
function del (cache, key, opts) {
return insert(cache, key, null, opts)
}
module.exports.delete.sync = delSync
function delSync (cache, key, opts) {
return insertSync(cache, key, null, opts)
}
module.exports.lsStream = lsStream
function lsStream (cache) {
const indexDir = bucketDir(cache)
@@ -108,7 +167,6 @@ function lsStream (cache) {
// "/cachename/<bucket 0xFF>/<bucket 0xFF>/*"
return readdirOrEmpty(subbucketPath).map(entry => {
const getKeyToEntry = bucketEntries(
cache,
path.join(subbucketPath, entry)
).reduce((acc, entry) => {
acc.set(entry.key, entry)
@@ -144,32 +202,39 @@ function ls (cache) {
})
}
function bucketEntries (cache, bucket, filter) {
function bucketEntries (bucket, filter) {
return readFileAsync(
bucket, 'utf8'
).then(data => {
let entries = []
data.split('\n').forEach(entry => {
if (!entry) { return }
const pieces = entry.split('\t')
if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
// Hash is no good! Corruption or malice? Doesn't matter!
// EJECT EJECT
return
}
let obj
try {
obj = JSON.parse(pieces[1])
} catch (e) {
// Entry is corrupted!
return
}
if (obj) {
entries.push(obj)
}
})
return entries
).then(data => _bucketEntries(data, filter))
}
function bucketEntriesSync (bucket, filter) {
const data = fs.readFileSync(bucket, 'utf8')
return _bucketEntries(data, filter)
}
function _bucketEntries (data, filter) {
let entries = []
data.split('\n').forEach(entry => {
if (!entry) { return }
const pieces = entry.split('\t')
if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
// Hash is no good! Corruption or malice? Doesn't matter!
// EJECT EJECT
return
}
let obj
try {
obj = JSON.parse(pieces[1])
} catch (e) {
// Entry is corrupted!
return
}
if (obj) {
entries.push(obj)
}
})
return entries
}
module.exports._bucketDir = bucketDir
@@ -197,9 +262,9 @@ function hashEntry (str) {
function hash (str, digest) {
return crypto
.createHash(digest)
.update(str)
.digest('hex')
.createHash(digest)
.update(str)
.digest('hex')
}
function formatEntry (cache, entry) {
@@ -217,8 +282,8 @@ function formatEntry (cache, entry) {
function readdirOrEmpty (dir) {
return readdirAsync(dir)
.catch({code: 'ENOENT'}, () => [])
.catch({code: 'ENOTDIR'}, () => [])
.catch({code: 'ENOENT'}, () => [])
.catch({code: 'ENOTDIR'}, () => [])
}
function nop () {
+46
View File
@@ -31,6 +31,34 @@ function fixOwner (filepath, uid, gid) {
)
}
module.exports.chownr.sync = fixOwnerSync
function fixOwnerSync (filepath, uid, gid) {
if (!process.getuid) {
// This platform doesn't need ownership fixing
return
}
if (typeof uid !== 'number' && typeof gid !== 'number') {
// There's no permissions override. Nothing to do here.
return
}
if ((typeof uid === 'number' && process.getuid() === uid) &&
(typeof gid === 'number' && process.getgid() === gid)) {
// No need to override if it's already what we used.
return
}
try {
chownr.sync(
filepath,
typeof uid === 'number' ? uid : process.getuid(),
typeof gid === 'number' ? gid : process.getgid()
)
} catch (err) {
if (err.code === 'ENOENT') {
return null
}
}
}
module.exports.mkdirfix = mkdirfix
function mkdirfix (p, uid, gid, cb) {
return mkdirp(p).then(made => {
@@ -42,3 +70,21 @@ function mkdirfix (p, uid, gid, cb) {
return fixOwner(p, uid, gid).then(() => null)
})
}
module.exports.mkdirfix.sync = mkdirfixSync
function mkdirfixSync (p, uid, gid) {
try {
const made = mkdirp.sync(p)
if (made) {
fixOwnerSync(made, uid, gid)
return made
}
} catch (err) {
if (err.code === 'EEXIST') {
fixOwnerSync(p, uid, gid)
return null
} else {
throw err
}
}
}
+10 -2
View File
@@ -2,14 +2,21 @@
const BB = require('bluebird')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./fix-owner')
const path = require('path')
const rimraf = BB.promisify(require('rimraf'))
const uniqueFilename = require('unique-filename')
const TmpOpts = figgyPudding({
tmpPrefix: {},
uid: {},
gid: {}
})
module.exports.mkdir = mktmpdir
function mktmpdir (cache, opts) {
opts = opts || {}
opts = TmpOpts(opts)
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
return fixOwner.mkdirfix(tmpTarget, opts.uid, opts.gid).then(() => {
return tmpTarget
@@ -22,11 +29,12 @@ function withTmp (cache, opts, cb) {
cb = opts
opts = null
}
opts = opts || {}
opts = TmpOpts(opts)
return BB.using(mktmpdir(cache, opts).disposer(rimraf), cb)
}
module.exports.fix = fixtmpdir
function fixtmpdir (cache, opts) {
opts = TmpOpts(opts)
return fixOwner(path.join(cache, 'tmp'), opts.uid, opts.gid)
}
+27 -13
View File
@@ -3,6 +3,7 @@
const BB = require('bluebird')
const contentPath = require('./content/path')
const figgyPudding = require('figgy-pudding')
const finished = BB.promisify(require('mississippi').finished)
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
@@ -14,10 +15,22 @@ const ssri = require('ssri')
BB.promisifyAll(fs)
const VerifyOpts = figgyPudding({
concurrency: {
default: 20
},
filter: {},
log: {
default: { silly () {} }
},
uid: {},
gid: {}
})
module.exports = verify
function verify (cache, opts) {
opts = opts || {}
opts.log && opts.log.silly('verify', 'verifying cache at', cache)
opts = VerifyOpts(opts)
opts.log.silly('verify', 'verifying cache at', cache)
return BB.reduce([
markStartTime,
fixPerms,
@@ -40,7 +53,7 @@ function verify (cache, opts) {
})
}, {}).tap(stats => {
stats.runTime.total = stats.endTime - stats.startTime
opts.log && opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`)
})
}
@@ -53,7 +66,7 @@ function markEndTime (cache, opts) {
}
function fixPerms (cache, opts) {
opts.log && opts.log.silly('verify', 'fixing cache permissions')
opts.log.silly('verify', 'fixing cache permissions')
return fixOwner.mkdirfix(cache, opts.uid, opts.gid).then(() => {
// TODO - fix file permissions too
return fixOwner.chownr(cache, opts.uid, opts.gid)
@@ -70,11 +83,11 @@ function fixPerms (cache, opts) {
// 5. If content is not marked as live, rimraf it.
//
function garbageCollect (cache, opts) {
opts.log && opts.log.silly('verify', 'garbage collecting content')
opts.log.silly('verify', 'garbage collecting content')
const indexStream = index.lsStream(cache)
const liveContent = new Set()
indexStream.on('data', entry => {
if (opts && opts.filter && !opts.filter(entry)) { return }
if (opts.filter && !opts.filter(entry)) { return }
liveContent.add(entry.integrity.toString())
})
return finished(indexStream).then(() => {
@@ -117,7 +130,7 @@ function garbageCollect (cache, opts) {
})
})
}
}, {concurrency: opts.concurrency || 20}))
}, {concurrency: opts.concurrency}))
})
})
}
@@ -141,7 +154,7 @@ function verifyContent (filepath, sri) {
}
function rebuildIndex (cache, opts) {
opts.log && opts.log.silly('verify', 'rebuilding index')
opts.log.silly('verify', 'rebuilding index')
return index.ls(cache).then(entries => {
const stats = {
missingContent: 0,
@@ -153,7 +166,7 @@ function rebuildIndex (cache, opts) {
if (entries.hasOwnProperty(k)) {
const hashed = index._hashKey(k)
const entry = entries[k]
const excluded = opts && opts.filter && !opts.filter(entry)
const excluded = opts.filter && !opts.filter(entry)
excluded && stats.rejectedEntries++
if (buckets[hashed] && !excluded) {
buckets[hashed].push(entry)
@@ -170,7 +183,7 @@ function rebuildIndex (cache, opts) {
}
return BB.map(Object.keys(buckets), key => {
return rebuildBucket(cache, buckets[key], stats, opts)
}, {concurrency: opts.concurrency || 20}).then(() => stats)
}, {concurrency: opts.concurrency}).then(() => stats)
})
}
@@ -184,7 +197,8 @@ function rebuildBucket (cache, bucket, stats, opts) {
return index.insert(cache, entry.key, entry.integrity, {
uid: opts.uid,
gid: opts.gid,
metadata: entry.metadata
metadata: entry.metadata,
size: entry.size
}).then(() => { stats.totalEntries++ })
}).catch({code: 'ENOENT'}, () => {
stats.rejectedEntries++
@@ -195,13 +209,13 @@ function rebuildBucket (cache, bucket, stats, opts) {
}
function cleanTmp (cache, opts) {
opts.log && opts.log.silly('verify', 'cleaning tmp directory')
opts.log.silly('verify', 'cleaning tmp directory')
return rimraf(path.join(cache, 'tmp'))
}
function writeVerifile (cache, opts) {
const verifile = path.join(cache, '_lastverified')
opts.log && opts.log.silly('verify', 'writing verifile to ' + verifile)
opts.log.silly('verify', 'writing verifile to ' + verifile)
return fs.writeFileAsync(verifile, '' + (+(new Date())))
}