Browse Source

Split in packages + add dry run + add noerror test

master
Ezwen 5 months ago
parent
commit
0f8182834d
  1. 3
      src/main/kotlin/bandcampcollectiondownloader/BandCampDownloaderError.kt
  2. 45
      src/main/kotlin/bandcampcollectiondownloader/Main.kt
  3. 6
      src/main/kotlin/bandcampcollectiondownloader/core/Args.kt
  4. 3
      src/main/kotlin/bandcampcollectiondownloader/core/BandCampDownloaderError.kt
  5. 7
      src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt
  6. 48
      src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt
  7. 2
      src/main/kotlin/bandcampcollectiondownloader/core/Constants.kt
  8. 3
      src/main/kotlin/bandcampcollectiondownloader/core/CookiesManagement.kt
  9. 50
      src/main/kotlin/bandcampcollectiondownloader/main/Main.kt
  10. 71
      src/main/kotlin/bandcampcollectiondownloader/util/DryIO.kt
  11. 20
      src/main/kotlin/bandcampcollectiondownloader/util/IO.kt
  12. 112
      src/main/kotlin/bandcampcollectiondownloader/util/RealIO.kt
  13. 67
      src/main/kotlin/bandcampcollectiondownloader/util/Util.kt
  14. 38
      src/test/kotlin/bandcampcollectiodownloader/test/SystemTests.kt
  15. 448
      test-data/bcdtestcookies.json

3
src/main/kotlin/bandcampcollectiondownloader/BandCampDownloaderError.kt

@ -1,3 +0,0 @@
package bandcampcollectiondownloader
class BandCampDownloaderError(s: String) : Exception(s)

45
src/main/kotlin/bandcampcollectiondownloader/Main.kt

@ -1,45 +0,0 @@
package bandcampcollectiondownloader
import picocli.CommandLine
fun main(args: Array<String>) {
// Parsing args
System.setProperty("picocli.usage.width", Constants.LINESIZE.toString())
val parsedArgs: Args =
try {
CommandLine.populateCommand(Args(), *args)
}
// If the wrong arguments are given, show help + error message
catch (e: CommandLine.ParameterException) {
CommandLine.usage(Args(), System.out)
System.err.println(e.message)
return
}
// If --version, then only shows version and quit
if (parsedArgs.version) {
println(Constants.VERSION)
return
}
// If --help, then only show help and quit
if (parsedArgs.help) {
CommandLine.usage(Args(), System.out)
}
// Else, parse arguments and run
else {
try {
BandcampCollectionDownloader.downloadAll(parsedArgs)
} catch (e: BandCampDownloaderError) {
System.err.println("ERROR: ${e.message}")
}
}
}

6
src/main/kotlin/bandcampcollectiondownloader/Args.kt → src/main/kotlin/bandcampcollectiondownloader/core/Args.kt

@ -1,4 +1,4 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.core
import picocli.CommandLine
import java.nio.file.Path
@ -41,6 +41,8 @@ data class Args(
var jobs: Int = 4,
@CommandLine.Option(names = ["-v", "--version"], versionHelp = true, description = ["Display the version and exits."])
var version: Boolean = false
var version: Boolean = false,
@CommandLine.Option(names = ["-n", "--dry-run"], usageHelp = false, description = ["Perform a trial run with no changes made."])
var dryRun: Boolean = false
)

3
src/main/kotlin/bandcampcollectiondownloader/core/BandCampDownloaderError.kt

@ -0,0 +1,3 @@
package bandcampcollectiondownloader.core
class BandCampDownloaderError(s: String) : Exception(s)

7
src/main/kotlin/bandcampcollectiondownloader/BandcampAPIConnector.kt → src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt

@ -1,5 +1,6 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.core
import bandcampcollectiondownloader.util.Util
import com.google.gson.Gson
import org.jsoup.Connection.Method
import org.jsoup.HttpStatusException
@ -18,8 +19,8 @@ class BandcampAPIConnector constructor(private val bandcampUser: String, private
private var initialized: Boolean = false
private data class ParsedFanpageData(
val fan_data: FanData,
val collection_data: CollectionData
val fan_data: FanData,
val collection_data: CollectionData
)
private data class FanData(

48
src/main/kotlin/bandcampcollectiondownloader/BandcampCollectionDownloader.kt → src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt

@ -1,6 +1,7 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.core
import org.zeroturnaround.zip.ZipUtil
import bandcampcollectiondownloader.util.IO
import bandcampcollectiondownloader.util.Util
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
@ -12,21 +13,21 @@ import java.util.*
import java.util.concurrent.*
import java.util.stream.Collectors
object BandcampCollectionDownloader {
class BandcampCollectionDownloader(private val args: Args, private val io: IO) {
class Cache constructor(private val path: Path) {
class Cache constructor(private val path: Path, private val io: IO) {
fun getContent(): List<String> {
if (!path.toFile().exists()) {
return emptyList()
}
return path.toFile().readLines().map { line -> line.split("|")[0] }
return io.readLines(path).map { line -> line.split("|")[0] }
}
fun add(id: String, description: String) {
if (!Files.exists(path)) {
Files.createFile(path)
io.createFile(path)
}
path.toFile().appendText("$id| $description\n")
io.append(path,"$id| $description\n")
}
}
@ -34,14 +35,16 @@ object BandcampCollectionDownloader {
/**
* Core function called from the Main function.
*/
fun downloadAll(args: Args) {
fun downloadAll(): Boolean {
Util.log("Target Bandcamp account: " + args.bandcampUser)
Util.log("Target download folder: " + args.pathToDownloadFolder.toAbsolutePath().normalize())
Util.log("Target audio format: " + args.audioFormat)
Util.logSeparator()
var result = true
// Gather cookies
val cookiesCandidates : MutableList<CookiesManagement.Cookies> = ArrayList()
val cookiesCandidates: MutableList<CookiesManagement.Cookies> = ArrayList()
if (args.pathToCookiesFile != null) {
// Parse JSON cookies (obtained with "Cookie Quick Manager" Firefox addon)
Util.log("Loading provided cookies file…")
@ -82,7 +85,7 @@ object BandcampCollectionDownloader {
// Prepare/load cache file
val cacheFilePath = args.pathToDownloadFolder.resolve("bandcamp-collection-downloader.cache")
val cache = Cache(cacheFilePath)
val cache = Cache(cacheFilePath, io)
val cacheContent = cache.getContent()
// Only work on items that have not been downloaded yet
@ -111,9 +114,10 @@ object BandcampCollectionDownloader {
Util.log("Managing item $itemNumber/${itemsToDownload.size}")
try {
manageDownloadPage(connector, saleItemID, args, cache)
manageDownloadPage(connector, saleItemID, cache)
} catch (e: BandCampDownloaderError) {
Util.log("Could not download item: " + e.message)
result = false
}
}
@ -125,11 +129,15 @@ object BandcampCollectionDownloader {
}
}
// To make sure we quit once all is done
threadPoolExecutor.shutdown()
threadPoolExecutor.awaitTermination(1, TimeUnit.DAYS)
return result
}
private fun manageDownloadPage(connector: BandcampAPIConnector, saleItemId: String, args: Args, cache: Cache) {
private fun manageDownloadPage(connector: BandcampAPIConnector, saleItemId: String, cache: Cache) {
val digitalItem = connector.retrieveDigitalItemData(saleItemId)
@ -258,36 +266,38 @@ object BandcampCollectionDownloader {
): Boolean {
// If the artist folder does not exist, we create it
if (!Files.exists(artistFolderPath)) {
Files.createDirectories(artistFolderPath)
io.createDirectories(artistFolderPath)
}
// If the release folder does not exist, we create it
if (!Files.exists(releaseFolderPath)) {
Files.createDirectories(releaseFolderPath)
io.createDirectories(releaseFolderPath)
}
// If the folder is empty, or if it only contains the zip.part file, we proceed
val amountFiles = releaseFolderPath.toFile().listFiles()!!.size
val amountFiles =
if (!args.dryRun || Files.exists(releaseFolderPath)) releaseFolderPath.toFile().listFiles()!!.size
else 0
if (amountFiles < 2) {
// Download content
val outputFilePath: Path = Util.downloadFile(fileURL, releaseFolderPath, timeout = timeout)
val outputFilePath: Path = io.downloadFile(fileURL, releaseFolderPath, timeout = timeout)
// If this is a zip, we unzip
if (!isSingleTrack) {
// Unzip
try {
ZipUtil.unpack(outputFilePath.toFile(), releaseFolderPath.toFile())
io.unzip(outputFilePath, releaseFolderPath)
} finally {
// Delete zip
Files.delete(outputFilePath)
io.delete(outputFilePath)
}
}
// Else if this is a single track, we just fetch the cover
else {
Util.downloadFile(coverURL, releaseFolderPath, "cover.jpg", timeout)
io.downloadFile(coverURL, releaseFolderPath, "cover.jpg", timeout)
}
return true
} else {

2
src/main/kotlin/bandcampcollectiondownloader/Constants.kt → src/main/kotlin/bandcampcollectiondownloader/core/Constants.kt

@ -1,4 +1,4 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.core
object Constants {

3
src/main/kotlin/bandcampcollectiondownloader/CookiesManagement.kt → src/main/kotlin/bandcampcollectiondownloader/core/CookiesManagement.kt

@ -1,5 +1,6 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.core
import bandcampcollectiondownloader.util.Util
import com.google.gson.Gson
import com.google.gson.JsonSyntaxException
import com.google.gson.annotations.SerializedName

50
src/main/kotlin/bandcampcollectiondownloader/main/Main.kt

@ -0,0 +1,50 @@
package bandcampcollectiondownloader.core
import bandcampcollectiondownloader.util.DryIO
import bandcampcollectiondownloader.util.IO
import bandcampcollectiondownloader.util.RealIO
import picocli.CommandLine
fun main(args: Array<String>) {
// Parsing args
System.setProperty("picocli.usage.width", Constants.LINESIZE.toString())
val parsedArgs: Args =
try {
CommandLine.populateCommand(Args(), *args)
}
// If the wrong arguments are given, show help + error message
catch (e: CommandLine.ParameterException) {
CommandLine.usage(Args(), System.out)
System.err.println(e.message)
return
}
// If --version, then only shows version and quit
if (parsedArgs.version) {
println(Constants.VERSION)
return
}
// If --help, then only show help and quit
if (parsedArgs.help) {
CommandLine.usage(Args(), System.out)
}
// Else, parse arguments and run
else {
try {
val io: IO =
if (parsedArgs.dryRun) DryIO() else RealIO()
BandcampCollectionDownloader(parsedArgs, io).downloadAll()
} catch (e: BandCampDownloaderError) {
System.err.println("ERROR: ${e.message}")
}
}
}

71
src/main/kotlin/bandcampcollectiondownloader/util/DryIO.kt

@ -0,0 +1,71 @@
package bandcampcollectiondownloader.util
import bandcampcollectiondownloader.core.BandCampDownloaderError
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
class DryIO : IO {
private val createdFiles: MutableList<Path> = ArrayList()
private fun exists(path: Path): Boolean {
return Files.exists(path) || !createdFiles.none { p -> p == path }
}
private fun existsOrError(path: Path) {
if (!exists(path)) throw BandCampDownloaderError(
"File cannot exist at this point: " + path.toAbsolutePath().normalize().toString()
)
}
private fun log(message: String) {
Util.log("""[dry run] Would $message""")
}
override fun createFile(path: Path) {
log("""create file ${path.toAbsolutePath().normalize()}""")
createdFiles.add(path)
}
override fun append(path: Path, content: String) {
log("""append "${content.trim()}" to file ${path.toAbsolutePath().normalize()}""")
existsOrError(path)
}
override fun createDirectories(path: Path) {
log("""create directories for path ${path.toAbsolutePath().normalize()}""")
createdFiles.add(path)
}
override fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String, timeout: Int): Path {
log("""download file from $fileURL to folder $saveDir""")
existsOrError(saveDir)
val fakeFile = saveDir.resolve("""unknownDownloadedFileName${Random().nextInt()}""")
createdFiles.add(fakeFile)
return fakeFile
}
override fun delete(path: Path) {
log("""would delete ${path.toAbsolutePath().normalize()}""")
existsOrError(path)
createdFiles.removeIf { p ->
p.toAbsolutePath().normalize().toString().equals(path.toAbsolutePath().normalize())
}
}
override fun unzip(zipPath: Path, outputFolderPath: Path) {
log(
"""would unzip file ${zipPath.toAbsolutePath().normalize()} into folder ${
outputFolderPath.toAbsolutePath().normalize()
}"""
)
existsOrError(zipPath)
existsOrError(outputFolderPath)
}
override fun readLines(path: Path): List<String> {
existsOrError(path)
return emptyList()
}
}

20
src/main/kotlin/bandcampcollectiondownloader/util/IO.kt

@ -0,0 +1,20 @@
package bandcampcollectiondownloader.util
import java.nio.file.Path
interface IO {
fun createFile(path: Path)
fun append(path: Path, content: String)
fun createDirectories(path: Path)
fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = "", timeout: Int): Path
fun delete(path: Path)
fun unzip(zipPath: Path, outputFolderPath: Path)
fun readLines(path: Path) : List<String>
}

112
src/main/kotlin/bandcampcollectiondownloader/Util.kt → src/main/kotlin/bandcampcollectiondownloader/util/RealIO.kt

@ -1,5 +1,6 @@
package bandcampcollectiondownloader
package bandcampcollectiondownloader.util
import org.zeroturnaround.zip.ZipUtil
import java.io.FileOutputStream
import java.net.HttpURLConnection
import java.net.URL
@ -8,21 +9,32 @@ import java.nio.file.Path
import java.nio.file.Paths
import java.text.DecimalFormat
import java.util.concurrent.ConcurrentHashMap
import java.util.function.BiPredicate
import javax.mail.internet.ContentDisposition
import kotlin.streams.toList
class RealIO(dryRun: Boolean = false) : IO {
object Util {
private val dryRun : Boolean = dryRun
private const val BUFFER_SIZE = 4096
override fun createFile(path: Path) {
Files.createFile(path)
}
override fun append(path: Path, content: String) {
path.toFile().appendText(content)
}
override fun createDirectories(path: Path) {
Files.createDirectories(path)
}
private val BUFFER_SIZE = 4096
private val parallelDownloadsProgresses: MutableMap<String, String> = ConcurrentHashMap()
/**
* From http://www.codejava.net/java-se/networking/use-httpurlconnection-to-download-file-from-an-http-url
* Initially from http://www.codejava.net/java-se/networking/use-httpurlconnection-to-download-file-from-an-http-url
*/
fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String = "", timeout: Int): Path {
override fun downloadFile(fileURL: String, saveDir: Path, optionalFileName: String, timeout: Int): Path {
// Prepare HTTP connection
val url = URL(fileURL)
@ -37,14 +49,14 @@ object Util {
// Retrieve information (name, size)
val disposition = httpConn.getHeaderField("Content-Disposition")
val fileName: String =
when {
optionalFileName != "" -> optionalFileName
disposition != null -> {
val parsedDisposition = ContentDisposition(disposition)
parsedDisposition.getParameter("filename")
}
else -> Paths.get(url.file).fileName.toString()
when {
optionalFileName != "" -> optionalFileName
disposition != null -> {
val parsedDisposition = ContentDisposition(disposition)
parsedDisposition.getParameter("filename")
}
else -> Paths.get(url.file).fileName.toString()
}
val contentLengthHeaderField = httpConn.getHeaderField("Content-Length")
val fileSize: Long = contentLengthHeaderField.toLong()
@ -76,13 +88,13 @@ object Util {
// Print progresses of all download threads
val prefix =
if (parallelDownloadsProgresses.size > 1) {
"Progresses: "
} else {
"Progress: "
}
if (parallelDownloadsProgresses.size > 1) {
"Progresses: "
} else {
"Progress: "
}
val message = "$prefix$allProgresses"
print(fillWithBlanks(message) + "\r")
print(Util.fillWithBlanks(message) + "\r")
// Download a new chunk
outputStream.write(buffer, 0, bytesRead)
@ -95,7 +107,7 @@ object Util {
// Clean the console output if needed
if (parallelDownloadsProgresses.isEmpty())
print(fillWithBlanks("") + "\r")
print(Util.fillWithBlanks("") + "\r")
// Close streams and connections
outputStream.close()
@ -111,62 +123,16 @@ object Util {
}
fun isUnix(): Boolean {
val os = System.getProperty("os.name").toLowerCase()
return os.indexOf("nix") >= 0 || os.indexOf("nux") >= 0 || os.indexOf("aix") > 0
}
fun isWindows(): Boolean {
val os = System.getProperty("os.name").toLowerCase()
return os.indexOf("win") >= 0
override fun delete(path: Path) {
Files.delete(path)
}
fun replaceInvalidCharsByUnicode(s: String): String {
var result: String = s
for ((old, new) in Constants.UNICODE_CHARS_REPLACEMENTS) {
result = result.replace(old, new)
}
return result
}
override fun unzip(zipPath: Path, outputFolderPath: Path) {
ZipUtil.unpack(zipPath.toFile(), outputFolderPath.toFile())
private fun fillWithBlanks(message: String): String {
return message + " ".repeat((Constants.LINESIZE - message.length).coerceAtLeast(0))
}
fun log(message: String) {
var messageWithPrefix: String = message
if (Thread.currentThread().name != "main") {
messageWithPrefix = "[${Thread.currentThread().name}] " + messageWithPrefix
}
println(fillWithBlanks(messageWithPrefix))
override fun readLines(path: Path): List<String> {
return path.toFile().readLines()
}
fun logSeparator() {
log("------------")
}
fun <T> retry(function: () -> T, retries: Int, ignoreFailure: Boolean = false): T? {
// Download release, with as many retries as configured
val attempts = retries + 1
for (i in 1..attempts) {
if (i > 1) {
log("Retrying (${i - 1}/${retries}).")
Thread.sleep(1000)
}
try {
return function()
} catch (e: Throwable) {
log("""Error while trying: "${e.javaClass.name}: ${e.message}".""")
}
}
val message = "Could not perform task after $retries retries."
if (ignoreFailure) {
log(message)
return null
} else {
throw BandCampDownloaderError(message)
}
}
}

67
src/main/kotlin/bandcampcollectiondownloader/util/Util.kt

@ -0,0 +1,67 @@
package bandcampcollectiondownloader.util
import bandcampcollectiondownloader.core.BandCampDownloaderError
import bandcampcollectiondownloader.core.Constants
object Util {
fun isUnix(): Boolean {
val os = System.getProperty("os.name").toLowerCase()
return os.indexOf("nix") >= 0 || os.indexOf("nux") >= 0 || os.indexOf("aix") > 0
}
fun isWindows(): Boolean {
val os = System.getProperty("os.name").toLowerCase()
return os.indexOf("win") >= 0
}
fun replaceInvalidCharsByUnicode(s: String): String {
var result: String = s
for ((old, new) in Constants.UNICODE_CHARS_REPLACEMENTS) {
result = result.replace(old, new)
}
return result
}
fun fillWithBlanks(message: String): String {
return message + " ".repeat((Constants.LINESIZE - message.length).coerceAtLeast(0))
}
fun log(message: String) {
var messageWithPrefix: String = message
if (Thread.currentThread().name != "main") {
messageWithPrefix = "[${Thread.currentThread().name}] " + messageWithPrefix
}
println(fillWithBlanks(messageWithPrefix))
}
fun logSeparator() {
log("------------")
}
fun <T> retry(function: () -> T, retries: Int, ignoreFailure: Boolean = false): T? {
// Download release, with as many retries as configured
val attempts = retries + 1
for (i in 1..attempts) {
if (i > 1) {
log("Retrying (${i - 1}/${retries}).")
Thread.sleep(1000)
}
try {
return function()
} catch (e: Throwable) {
log("""Error while trying: "${e.javaClass.name}: ${e.message}".""")
}
}
val message = "Could not perform task after $retries retries."
if (ignoreFailure) {
log(message)
return null
} else {
throw BandCampDownloaderError(message)
}
}
}

38
src/test/kotlin/bandcampcollectiodownloader/test/BandcampCollectionDownloaderTests.kt → src/test/kotlin/bandcampcollectiodownloader/test/SystemTests.kt

@ -1,10 +1,13 @@
package bandcampcollectiodownloader.test
import bandcampcollectiondownloader.Args
import bandcampcollectiondownloader.BandCampDownloaderError
import bandcampcollectiondownloader.BandcampCollectionDownloader
import bandcampcollectiondownloader.core.Args
import bandcampcollectiondownloader.core.BandCampDownloaderError
import bandcampcollectiondownloader.core.BandcampCollectionDownloader
import bandcampcollectiondownloader.util.DryIO
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import java.nio.file.Files
import java.nio.file.Paths
import java.util.*
@ -12,14 +15,14 @@ import java.util.*
* Note: bli is a valid bandcamp user (completely randomly chosen),
* but for which we have no credentials (ie. valid cookies).
*/
class BandcampCollectionDownloaderTests {
class SystemTests {
@Test
fun testErrorCookiesFileNotFound() {
val args = Args()
args.pathToCookiesFile = Paths.get("bli")
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -29,7 +32,7 @@ class BandcampCollectionDownloaderTests {
val args = Args()
args.pathToCookiesFile = Paths.get("./test-data/notjsoncookies.json")
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -38,7 +41,7 @@ class BandcampCollectionDownloaderTests {
val args = Args()
args.pathToCookiesFile = Paths.get("./test-data/invalidcookies_wrongkeys.json")
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -47,7 +50,7 @@ class BandcampCollectionDownloaderTests {
val args = Args()
args.pathToCookiesFile = Paths.get("./test-data/invalidcookies_noarray.json")
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -58,7 +61,7 @@ class BandcampCollectionDownloaderTests {
args.bandcampUser = "zerz1e3687dfs3df7"
args.timeout = 5000
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -69,7 +72,7 @@ class BandcampCollectionDownloaderTests {
args.bandcampUser = "bli"
args.timeout = 5000
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@ -81,10 +84,23 @@ class BandcampCollectionDownloaderTests {
args.bandcampUser = "bli"
args.timeout = 5000
assertThrows<BandCampDownloaderError> {
BandcampCollectionDownloader.downloadAll(args)
BandcampCollectionDownloader(args, DryIO()).downloadAll()
}
}
@Test
fun testOKValidCookiesDryRun() {
val args = Args()
args.pathToCookiesFile = Paths.get("./test-data/bcdtestcookies.json")
args.bandcampUser = "bcdtest"
val tmpDir = Files.createTempDirectory("bandcamp-collection-downloader-test")
tmpDir.toFile().deleteOnExit()
args.pathToDownloadFolder = tmpDir
args.dryRun = true
val result = BandcampCollectionDownloader(args, DryIO()).downloadAll()
assertTrue(result)
}
@Throws(Exception::class)
fun addToEnv(key: String, value: String) {
val classes = Collections::class.java.declaredClasses

448
test-data/bcdtestcookies.json

@ -0,0 +1,448 @@
[{
"Host raw": "https://.bandcamp.com/",
"Name raw": "client_id",
"Path raw": "/",
"Content raw": "37B4B358043DF16DF6B72FF7304D609A1D5866AAD8CB12873870C66CF026B5EF",
"Expires": "16-12-2030 13:52:20",
"Expires raw": "1923655940",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "true",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "_comm_playlist_48947135",
"Path raw": "/",
"Content raw": "%7B%22last%22%3A6%2C%22msgs%22%3A%5B%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%5D%7D",
"Expires": "16-12-2020 13:53:46",
"Expires raw": "1608123226",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "_comm_playlist_481921806",
"Path raw": "/",
"Content raw": "%7B%22last%22%3A7%2C%22msgs%22%3A%5B%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%2C%5B%22stop%22%5D%5D%7D",
"Expires": "16-12-2020 14:53:29",
"Expires raw": "1608126809",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "BACKENDID",
"Path raw": "/",
"Content raw": "bender28-6",
"Expires": "At the end of the session",
"Expires raw": "0",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "unique_24h",
"Path raw": "/",
"Content raw": "223",
"Expires": "24-12-2020 13:34:36",
"Expires raw": "1608813276",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "_comm_fan_verification_226224519",
"Path raw": "/",
"Content raw": "%7B%22last%22%3A1%2C%22msgs%22%3A%5B%5B%22verified%22%2C1%5D%5D%7D",
"Expires": "23-12-2020 13:34:43",
"Expires raw": "1608726883",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "fan_visits",
"Path raw": "/",
"Content raw": "107042",
"Expires": "15-12-2027 13:55:20",
"Expires raw": "1828875320",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "identity",
"Path raw": "/",
"Content raw": "7%09BEw3Ng0phv59zTf8ZPyJwQBdIhGi3BDY9x9BHX0YKoU%3D%09%7B%22ex%22%3A0%2C%22id%22%3A1774328967%7D",
"Expires": "15-12-2027 13:55:30",
"Expires raw": "1828875330",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "true",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "js_logged_in",
"Path raw": "/",
"Content raw": "1",
"Expires": "15-12-2027 13:55:30",
"Expires raw": "1828875330",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "logout",
"Path raw": "/",
"Content raw": "%7B%22username%22%3A%22bcdtest%22%7D",
"Expires": "15-12-2027 13:55:30",
"Expires raw": "1828875330",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "download_encoding",
"Path raw": "/",
"Content raw": "103",
"Expires": "15-12-2027 13:58:59",
"Expires raw": "1828875539",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "session",
"Path raw": "/",
"Content raw": "1%09c%3A1%09bp%3A1%09t%3A1608719181%09r%3A%5B%22nilZ0f0x1608733633%22%2C%22nilZ0c0x1608728340%22%2C%22nilZ0a4071882345x1608728234%22%5D",
"Expires": "06-02-2021 11:26:21",
"Expires raw": "1612607181",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-default",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "client_id",
"Path raw": "/",
"Content raw": "C2C2181CE1DC3A9CCD6142574A08C767EE721D5C050639AED9002A13B25E898D",
"Expires": "23-12-2030 13:59:52",
"Expires raw": "1924261192",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "true",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-2",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "session",
"Path raw": "/",
"Content raw": "1%09t%3A1608728392%09r%3A%5B%22nilZ0c0x1608728392%22%5D",
"Expires": "06-02-2021 13:59:52",
"Expires raw": "1612616392",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-2",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "fan_visits",
"Path raw": "/",
"Content raw": "6483105",
"Expires": "15-12-2027 13:59:52",
"Expires raw": "1828875592",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-2",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "BACKENDID",
"Path raw": "/",
"Content raw": "red-78b9-7",
"Expires": "At the end of the session",
"Expires raw": "0",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-2",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "client_id",
"Path raw": "/",
"Content raw": "44D0C15C6B5FE4FC9280CCD6B53C9DF0763F6C14AC333CAC0AC5D10AD32FBE84",
"Expires": "23-12-2030 13:32:58",
"Expires raw": "1924259578",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "true",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "unique_24h",
"Path raw": "/",
"Content raw": "223",
"Expires": "24-12-2020 13:32:58",
"Expires raw": "1608813178",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "BACKENDID",
"Path raw": "/",
"Content raw": "bender23-3",
"Expires": "At the end of the session",
"Expires raw": "0",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "fan_visits",
"Path raw": "/",
"Content raw": "6483105",
"Expires": "15-12-2027 13:34:47",
"Expires raw": "1828874087",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "identity",
"Path raw": "/",
"Content raw": "7%09BEw3Ng0phv59zTf8ZPyJwQBdIhGi3BDY9x9BHX0YKoU%3D%09%7B%22ex%22%3A0%2C%22id%22%3A1774328967%7D",
"Expires": "15-12-2027 13:34:59",
"Expires raw": "1828874099",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "true",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "https://.bandcamp.com/",
"Name raw": "js_logged_in",
"Path raw": "/",
"Content raw": "1",
"Expires": "15-12-2027 13:34:59",
"Expires raw": "1828874099",
"Send for": "Encrypted connections only",
"Send for raw": "true",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "logout",
"Path raw": "/",
"Content raw": "%7B%22username%22%3A%22bcdtest%22%7D",
"Expires": "15-12-2027 13:34:59",
"Expires raw": "1828874099",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "_comm_fan_verification_549577207",
"Path raw": "/",
"Content raw": "%7B%22last%22%3A1%2C%22msgs%22%3A%5B%5B%22verified%22%2C1%5D%5D%7D",
"Expires": "23-12-2020 13:35:17",
"Expires raw": "1608726917",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "download_encoding",
"Path raw": "/",
"Content raw": "103",
"Expires": "15-12-2027 13:53:04",
"Expires raw": "1828875184",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "session",
"Path raw": "/",
"Content raw": "1%09c%3A1%09bp%3A1%09t%3A1608726778%09r%3A%5B%22nilZ0c0x1608728155%22%2C%22289772975s0a4071882345x1608727646%22%2C%22229967901s0a2813015147x1608727627%22%5D",
"Expires": "06-02-2021 13:32:58",
"Expires raw": "1612614778",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "no_restriction",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "__stripe_mid",
"Path raw": "/",
"Content raw": "c650f7c6-0396-4074-8e24-a728585e452fd839f4",
"Expires": "23-12-2021 13:56:22",
"Expires raw": "1640264182",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "lax",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
},
{
"Host raw": "http://.bandcamp.com/",
"Name raw": "__stripe_sid",
"Path raw": "/",
"Content raw": "76f89884-54be-4c02-9399-c03b4fdd466ecf7215",
"Expires": "23-12-2020 14:26:22",
"Expires raw": "1608729982",
"Send for": "Any type of connection",
"Send for raw": "false",
"HTTP only raw": "false",
"SameSite raw": "lax",
"This domain only": "Valid for subdomains",
"This domain only raw": "false",
"Store raw": "firefox-container-4",
"First Party Domain": ""
}]
Loading…
Cancel
Save