mirror of
https://github.com/gedoor/legado.git
synced 2024-08-30 09:23:26 +08:00
添加多页目录并发访问限制
This commit is contained in:
kunfei
parent
20c2daeadb
commit
118be1d2e4
@ -3,9 +3,9 @@ package io.legado.app.model.analyzeRule
|
||||
import android.annotation.SuppressLint
|
||||
import android.util.Base64
|
||||
import androidx.annotation.Keep
|
||||
import cn.hutool.core.util.HexUtil
|
||||
import com.bumptech.glide.load.model.GlideUrl
|
||||
import com.script.SimpleBindings
|
||||
import cn.hutool.core.util.HexUtil
|
||||
import io.legado.app.constant.AppConst.SCRIPT_ENGINE
|
||||
import io.legado.app.constant.AppConst.UA_NAME
|
||||
import io.legado.app.constant.AppPattern.JS_PATTERN
|
||||
@ -279,6 +279,7 @@ class AnalyzeUrl(
|
||||
/**
|
||||
* 开始访问,并发判断
|
||||
*/
|
||||
@Throws(ConcurrentException::class)
|
||||
private fun fetchStart(): ConcurrentRecord? {
|
||||
source ?: return null
|
||||
val concurrentRate = source.concurrentRate
|
||||
@ -294,10 +295,13 @@ class AnalyzeUrl(
|
||||
}
|
||||
val waitTime: Int = synchronized(fetchRecord) {
|
||||
try {
|
||||
if (rateIndex == -1) {
|
||||
if (!fetchRecord.isConcurrent) {
|
||||
//并发控制非 次数/毫秒
|
||||
if (fetchRecord.frequency > 0) {
|
||||
//已经有访问线程,直接等待
|
||||
return@synchronized concurrentRate.toInt()
|
||||
}
|
||||
//没有线程访问,判断还剩多少时间可以访问
|
||||
val nextTime = fetchRecord.time + concurrentRate.toInt()
|
||||
if (System.currentTimeMillis() >= nextTime) {
|
||||
fetchRecord.time = System.currentTimeMillis()
|
||||
@ -306,9 +310,11 @@ class AnalyzeUrl(
|
||||
}
|
||||
return@synchronized (nextTime - System.currentTimeMillis()).toInt()
|
||||
} else {
|
||||
//并发控制为 次数/毫秒
|
||||
val sj = concurrentRate.substring(rateIndex + 1)
|
||||
val nextTime = fetchRecord.time + sj.toInt()
|
||||
if (System.currentTimeMillis() >= nextTime) {
|
||||
//已经过了限制时间,重置开始时间
|
||||
fetchRecord.time = System.currentTimeMillis()
|
||||
fetchRecord.frequency = 1
|
||||
return@synchronized 0
|
||||
@ -335,7 +341,7 @@ class AnalyzeUrl(
|
||||
* 访问结束
|
||||
*/
|
||||
private fun fetchEnd(concurrentRecord: ConcurrentRecord?) {
|
||||
if (concurrentRecord != null && !concurrentRecord.concurrent) {
|
||||
if (concurrentRecord != null && !concurrentRecord.isConcurrent) {
|
||||
synchronized(concurrentRecord) {
|
||||
concurrentRecord.frequency = concurrentRecord.frequency - 1
|
||||
}
|
||||
@ -345,6 +351,7 @@ class AnalyzeUrl(
|
||||
/**
|
||||
* 访问网站,返回StrResponse
|
||||
*/
|
||||
@Throws(ConcurrentException::class)
|
||||
suspend fun getStrResponseAwait(
|
||||
jsStr: String? = null,
|
||||
sourceRegex: String? = null,
|
||||
@ -414,6 +421,7 @@ class AnalyzeUrl(
|
||||
}
|
||||
|
||||
@JvmOverloads
|
||||
@Throws(ConcurrentException::class)
|
||||
fun getStrResponse(
|
||||
jsStr: String? = null,
|
||||
sourceRegex: String? = null,
|
||||
@ -427,6 +435,7 @@ class AnalyzeUrl(
|
||||
/**
|
||||
* 访问网站,返回Response
|
||||
*/
|
||||
@Throws(ConcurrentException::class)
|
||||
suspend fun getResponseAwait(): Response {
|
||||
val concurrentRecord = fetchStart()
|
||||
try {
|
||||
@ -457,6 +466,7 @@ class AnalyzeUrl(
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(ConcurrentException::class)
|
||||
fun getResponse(): Response {
|
||||
return runBlocking {
|
||||
getResponseAwait()
|
||||
@ -464,6 +474,7 @@ class AnalyzeUrl(
|
||||
}
|
||||
|
||||
@Suppress("UnnecessaryVariable")
|
||||
@Throws(ConcurrentException::class)
|
||||
private fun getByteArrayIfDataUri(): ByteArray? {
|
||||
@Suppress("RegExpRedundantEscape")
|
||||
val dataUriFindResult = dataUriRegex.find(urlNoQuery)
|
||||
@ -480,6 +491,7 @@ class AnalyzeUrl(
|
||||
* 访问网站,返回ByteArray
|
||||
*/
|
||||
@Suppress("UnnecessaryVariable", "LiftReturnOrAssignment")
|
||||
@Throws(ConcurrentException::class)
|
||||
suspend fun getByteArrayAwait(): ByteArray {
|
||||
getByteArrayIfDataUri()?.let {
|
||||
return it
|
||||
@ -497,6 +509,7 @@ class AnalyzeUrl(
|
||||
* 访问网站,返回InputStream
|
||||
*/
|
||||
@Suppress("LiftReturnOrAssignment")
|
||||
@Throws(ConcurrentException::class)
|
||||
suspend fun getInputStreamAwait(): InputStream {
|
||||
getByteArrayIfDataUri()?.let {
|
||||
return ByteArrayInputStream(it)
|
||||
@ -504,6 +517,7 @@ class AnalyzeUrl(
|
||||
return getResponseAwait().body!!.byteStream()
|
||||
}
|
||||
|
||||
@Throws(ConcurrentException::class)
|
||||
fun getInputStream(): InputStream {
|
||||
return runBlocking {
|
||||
getInputStreamAwait()
|
||||
@ -679,8 +693,17 @@ class AnalyzeUrl(
|
||||
}
|
||||
|
||||
data class ConcurrentRecord(
|
||||
val concurrent: Boolean,
|
||||
/**
|
||||
* 是否按频率
|
||||
*/
|
||||
val isConcurrent: Boolean,
|
||||
/**
|
||||
* 开始访问时间
|
||||
*/
|
||||
var time: Long,
|
||||
/**
|
||||
* 正在访问的个数
|
||||
*/
|
||||
var frequency: Int
|
||||
)
|
||||
|
||||
|
||||
@ -6,15 +6,18 @@ import io.legado.app.data.entities.Book
|
||||
import io.legado.app.data.entities.BookChapter
|
||||
import io.legado.app.data.entities.BookSource
|
||||
import io.legado.app.data.entities.rule.TocRule
|
||||
import io.legado.app.exception.ConcurrentException
|
||||
import io.legado.app.exception.NoStackTraceException
|
||||
import io.legado.app.exception.TocEmptyException
|
||||
import io.legado.app.help.book.ContentProcessor
|
||||
import io.legado.app.help.http.StrResponse
|
||||
import io.legado.app.model.Debug
|
||||
import io.legado.app.model.analyzeRule.AnalyzeRule
|
||||
import io.legado.app.model.analyzeRule.AnalyzeUrl
|
||||
import io.legado.app.utils.isTrue
|
||||
import kotlinx.coroutines.Dispatchers.IO
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.withContext
|
||||
import splitties.init.appCtx
|
||||
@ -61,12 +64,26 @@ object BookChapterList {
|
||||
var nextUrl = chapterData.second[0]
|
||||
while (nextUrl.isNotEmpty() && !nextUrlList.contains(nextUrl)) {
|
||||
nextUrlList.add(nextUrl)
|
||||
AnalyzeUrl(
|
||||
val analyzeUrl = AnalyzeUrl(
|
||||
mUrl = nextUrl,
|
||||
source = bookSource,
|
||||
ruleData = book,
|
||||
headerMapF = bookSource.getHeaderMap()
|
||||
).getStrResponseAwait().body?.let { nextBody ->
|
||||
)
|
||||
var res: StrResponse? = null
|
||||
var isConcurrent: Boolean
|
||||
do {
|
||||
//控制并发访问
|
||||
isConcurrent = false
|
||||
try {
|
||||
res = analyzeUrl.getStrResponseAwait()
|
||||
} catch (e: ConcurrentException) {
|
||||
isConcurrent = true
|
||||
//如果是并发限制等待再次访问
|
||||
delay(e.waitTime.toLong())
|
||||
}
|
||||
} while (!isConcurrent)
|
||||
res!!.body?.let { nextBody ->
|
||||
chapterData = analyzeChapterList(
|
||||
book, nextUrl, nextUrl,
|
||||
nextBody, tocRule, listRule, bookSource
|
||||
@ -89,10 +106,22 @@ object BookChapterList {
|
||||
ruleData = book,
|
||||
headerMapF = bookSource.getHeaderMap()
|
||||
)
|
||||
val res = analyzeUrl.getStrResponseAwait()
|
||||
var res: StrResponse? = null
|
||||
var isConcurrent: Boolean
|
||||
do {
|
||||
//控制并发访问
|
||||
isConcurrent = false
|
||||
try {
|
||||
res = analyzeUrl.getStrResponseAwait()
|
||||
} catch (e: ConcurrentException) {
|
||||
isConcurrent = true
|
||||
//如果是并发限制等待再次访问
|
||||
delay(e.waitTime.toLong())
|
||||
}
|
||||
} while (!isConcurrent)
|
||||
analyzeChapterList(
|
||||
book, urlStr, res.url,
|
||||
res.body!!, tocRule, listRule, bookSource, false
|
||||
book, urlStr, res!!.url,
|
||||
res!!.body!!, tocRule, listRule, bookSource, false
|
||||
).first
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user