Browse Source

add basic js disk cache with node-persist

main
James Fenn 5 months ago
parent
commit
03a8522b3e
13 changed files with 104 additions and 28 deletions
  1. +1
    -0
      .gitignore
  2. +1
    -1
      example-js/index.html
  3. +8
    -4
      example-nodejs/src/index.js
  4. +2
    -0
      gitrest/build.gradle
  5. +2
    -2
      gitrest/src/androidMain/kotlin/me/jfenn/gitrest/service/DiskCache.kt
  6. +2
    -2
      gitrest/src/androidMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt
  7. +2
    -2
      gitrest/src/commonMain/kotlin/me/jfenn/gitrest/service/Cache.kt
  8. +2
    -2
      gitrest/src/commonMain/kotlin/me/jfenn/gitrest/service/NoCache.kt
  9. +14
    -11
      gitrest/src/jsMain/kotlin/Client.kt
  10. +1
    -0
      gitrest/src/jsMain/kotlin/gitrest/js/Repo.kt
  11. +65
    -0
      gitrest/src/jsMain/kotlin/me/jfenn/gitrest/service/DiskCache.kt
  12. +2
    -2
      gitrest/src/jsMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt
  13. +2
    -2
      gitrest/src/jvmMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt

+ 1
- 0
.gitignore View File

@@ -12,6 +12,7 @@
.classpath
.settings/
.yarnrc
.node-persist/
yarn.lock
node_modules
/npm/index.js


+ 1
- 1
example-js/index.html View File

@@ -16,7 +16,7 @@
</body>
<script src="./gitrest.js"></script>
<script type="text/javascript">
let delegate = new gitrest.RequestProvider();
let delegate = new gitrest.Client();
delegate.getRepo("github@api.github.com:fennifith/Attribouter").then((repo) => {
console.log(repo);
document.getElementById("repoName").innerText = repo.id;


+ 8
- 4
example-nodejs/src/index.js View File

@@ -1,11 +1,15 @@
const { RequestProvider } = require("git-rest-wrapper-gitrest");
const { Client } = require("git-rest-wrapper-gitrest");

let delegate = new RequestProvider();
let gitrest = new Client({
cache: {
type: "disk"
}
});

delegate.getUser("fennifith").then((obj) => {
gitrest.getUser("fennifith").then((obj) => {
console.log(obj);
});

delegate.getRepo("gitlab@salsa.debian.org:reproducible-builds/reproducible-website").then((obj) => {
gitrest.getRepo("gitlab@salsa.debian.org:reproducible-builds/reproducible-website").then((obj) => {
console.log(obj);
});

+ 2
- 0
gitrest/build.gradle View File

@@ -147,6 +147,8 @@ kotlin {
implementation npm("node-fetch", "2.6.0")
implementation npm("text-encoding", "0.7.0")
implementation npm("abort-controller", "3.0.0")

implementation npm("node-persist", "3.1.0")
}
}
jsTest {


+ 2
- 2
gitrest/src/androidMain/kotlin/me/jfenn/gitrest/service/DiskCache.kt View File

@@ -25,7 +25,7 @@ class DiskCache(
fun String.cacheFile() = File(cacheDir, "${this.replace(File.separator, "_")}.json")

@ImplicitReflectionSerializer
override fun set(key: String, value: Any) {
override suspend fun set(key: String, value: Any) {
// obtain a serializer + type for the value (this is all just a ridiculous hack)
val serializer: KSerializer<Any>
val typeName: String
@@ -51,7 +51,7 @@ class DiskCache(
}

@ImplicitReflectionSerializer
override fun <T> get(key: String): T? {
override suspend fun <T> get(key: String): T? {
return try {
// check contents; destructure file parts if safe
val fileContents = key.cacheFile().readText().split("#", limit = 3)


+ 2
- 2
gitrest/src/androidMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt View File

@@ -20,7 +20,7 @@ actual class MemoryCache actual constructor(
private val referenceQueue = ReferenceQueue<Any>()
private val map = HashMap<String, WeakEntry>()

override fun set(key: String, value: Any) {
override suspend fun set(key: String, value: Any) {
// remove any invalid references
var weakEntry = referenceQueue.poll() as WeakEntry?
while (weakEntry != null) {
@@ -32,7 +32,7 @@ actual class MemoryCache actual constructor(
underlyingCache.set(key, value)
}

override fun <T> get(key: String): T? {
override suspend fun <T> get(key: String): T? {
val weakEntry = map[key]
weakEntry?.get()?.let {
return it as? T


+ 2
- 2
gitrest/src/commonMain/kotlin/me/jfenn/gitrest/service/Cache.kt View File

@@ -2,8 +2,8 @@ package me.jfenn.gitrest.service

interface Cache {

fun set(key: String, value: Any)
suspend fun set(key: String, value: Any)

fun <T> get(id: String) : T?
suspend fun <T> get(id: String) : T?

}

+ 2
- 2
gitrest/src/commonMain/kotlin/me/jfenn/gitrest/service/NoCache.kt View File

@@ -3,6 +3,6 @@ package me.jfenn.gitrest.service
import me.jfenn.gitrest.model.DelegateResource

object NoCache : Cache {
override fun set(key: String, value: Any) {}
override fun <T> get(id: String): T? = null
override suspend fun set(key: String, value: Any) {}
override suspend fun <T> get(id: String): T? = null
}

gitrest/src/jsMain/kotlin/RequestProvider.kt → gitrest/src/jsMain/kotlin/Client.kt View File

@@ -1,28 +1,31 @@
import me.jfenn.gitrest.base.ServiceBuilder
import me.jfenn.gitrest.provider.gitea.GiteaProvider
import me.jfenn.gitrest.provider.github.GithubProvider
import me.jfenn.gitrest.provider.gitlab.GitlabProvider
import gitrest.js.License
import gitrest.js.Repo
import gitrest.js.User
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.promise
import me.jfenn.gitrest.gitrest
import me.jfenn.gitrest.service.DiskCache
import kotlin.js.Promise

/**
* NodeJS-usable wrapper class to convert functions/arguments into their JS-compatible versions
*/
class RequestProvider(
providers: Array<ServiceBuilder<*>> = arrayOf(
GithubProvider,
GiteaProvider,
GitlabProvider
)
class Client(
config: dynamic
) {

val client = gitrest {
this.providers = providers
this.providers.forEach {
if (config.tokens != null) {
this.providers.forEach {
it.tokens.putAll(config.tokens)
}
}

if (config.cache?.type == "disk") {
this.cache = DiskCache(this)
}
}
}

@JsName("getUser")

+ 1
- 0
gitrest/src/jsMain/kotlin/gitrest/js/Repo.kt View File

@@ -14,5 +14,6 @@ class Repo(
val license: License? = repo.license?.let { License(it) }
val defaultBranch: String? = repo.defaultBranch

@JsName("getRawFileUrl")
fun getRawFileUrl(branchName: String, filePath: String) : String? = repo.getRawFileUrl(branchName, filePath)
}

+ 65
- 0
gitrest/src/jsMain/kotlin/me/jfenn/gitrest/service/DiskCache.kt View File

@@ -0,0 +1,65 @@
package me.jfenn.gitrest.service

import kotlinx.coroutines.await
import me.jfenn.gitrest.model.GitrestConfig
import me.jfenn.gitrest.model.Repo
import kotlin.js.Date
import kotlin.js.Promise

external fun require(module: String): dynamic

/**
* Unsafe "disk cache" based on node-persist and JSON.parse.
*/
class DiskCache(
val config: GitrestConfig,
val cacheDuration: Long = 864000000 // cache for ~10 days by default
) : Cache {

private var persist: dynamic = null
private var promisePersist: Promise<dynamic>? = null

private suspend fun getPersist() : dynamic {
if (persist == null) {
persist = require("node-persist")
promisePersist = (persist.init() as Promise<dynamic>)
}

promisePersist?.await()

return persist
}

/**
* Obtain a "version" string using properties of the Repo class
* - prop names are obfuscated, so this should be somewhat reliable.
*/
private fun getVersion() : String {
val keys: (dynamic) -> Array<String> = js("Object.keys")
return keys(Repo())[0]
}

override suspend fun set(key: String, value: Any) {
val string = getVersion() + "#" + Date.now() + "#" + JSON.stringify(value)
(getPersist().setItem(key, string) as Promise<dynamic>).await()
}

override suspend fun <T> get(key: String): T? {
val string = (getPersist().getItem(key) as Promise<String?>).await() ?: return null

val stringContents = string.split("#", limit = 3)
if (stringContents.size != 3) return null
val (version, lastModified, json) = stringContents

if (version != getVersion())
return null

if (Date.now() - lastModified.toLong() < cacheDuration)
return null

return if (json.startsWith("{"))
JSON.parse(json)
else null
}

}

+ 2
- 2
gitrest/src/jsMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt View File

@@ -11,12 +11,12 @@ actual class MemoryCache actual constructor(

private val map = HashMap<String, Any>()

override fun set(key: String, value: Any) {
override suspend fun set(key: String, value: Any) {
map[key] = value
underlyingCache.set(key, value)
}

override fun <T> get(key: String): T? {
override suspend fun <T> get(key: String): T? {
return (map[key] ?: underlyingCache.get<T>(key)) as? T?
}


+ 2
- 2
gitrest/src/jvmMain/kotlin/me/jfenn/gitrest/service/MemoryCache.kt View File

@@ -20,7 +20,7 @@ actual class MemoryCache actual constructor(
private val referenceQueue = ReferenceQueue<Any>()
private val map = HashMap<String, WeakEntry>()

override fun set(key: String, value: Any) {
override suspend fun set(key: String, value: Any) {
// remove any invalid references
var weakEntry = referenceQueue.poll() as WeakEntry?
while (weakEntry != null) {
@@ -32,7 +32,7 @@ actual class MemoryCache actual constructor(
underlyingCache.set(key, value)
}

override fun <T> get(key: String): T? {
override suspend fun <T> get(key: String): T? {
val weakEntry = map[key]
weakEntry?.get()?.let {
return it as? T


Loading…
Cancel
Save