mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 16:22:17 +00:00
2022.10.18.0
Moved UserMedia xml initialization to the structure itself Added download with feed skip Added silent mode (temporary disabling notifications) Added additional Instagram protection Excluding users whose profiles do not exist from downloading with groups and AutoDownloader Feed: delete file bugs; reorder data after file deletion; video playback bugs SiteSettingsForm: enable 'OK' button when editing cookies Fixed collection users ban Settings: disabling ffmpeg missing notification; advanced notification management Added 'ToolStripKeyMenuItem' control Plugins: deprecated XVIDEOS and LPSG plugin libraries; moved them to SCrawler. Updated license PluginProvider: added 'BeginEdit' and 'EndEdit' function to ISiteSettings; changed GetSpecialData (ISiteSettings) return type to IEnumerable PluginsEnvironment: removed 'IsMyClass' attribute MainFrame: grouped all download buttons into one menu; reorganized code; removed 'F2' hotkey AutoDownloader: added advanced pause options; added buttons to tray icon and AutoDownloader form MissingPosts: finished; activated functions that were disabled; added download functions to UserData classes UserDataBase: ability to use responser; ability to download m3u8; extended 'DownloadingException' with optional argument 'EObj'; user index in collection (button tag) changed to user instance; extended information with user labels; updated 'ProcessException' function Replaced download buttons with 'KeyClick' control Replaced FDatePickerForm with my library's form Collections: Deleting multiple collections - disabled confirmation; ban each user in collection
This commit is contained in:
Andy
104
SCrawler/API/LPSG/UserData.vb
Normal file
104
SCrawler/API/LPSG/UserData.vb
Normal file
@@ -0,0 +1,104 @@
|
||||
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Imports Converters = PersonalUtilities.Functions.SymbolsConverter.Converters
|
||||
Namespace API.LPSG
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
Private Const Name_LatestPage As String = "LatestPage"
|
||||
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
|
||||
If Loading Then
|
||||
LatestPage = Container.Value(Name_LatestPage)
|
||||
Else
|
||||
Container.Add(Name_LatestPage, LatestPage)
|
||||
End If
|
||||
End Sub
|
||||
Private Property LatestPage As String = String.Empty
|
||||
Private Enum Mode : Internal : External : End Enum
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
Responser.Error = EDP.ThrowException
|
||||
|
||||
Dim NextPage$
|
||||
Dim r$
|
||||
Dim _LPage As Func(Of String) = Function() If(LatestPage.IsEmptyString, String.Empty, $"page-{LatestPage}")
|
||||
|
||||
Do
|
||||
URL = $"https://www.lpsg.com/threads/{Name}/{_LPage.Invoke}"
|
||||
r = Responser.GetResponse(URL)
|
||||
UserExists = True
|
||||
UserSuspended = False
|
||||
ThrowAny(Token)
|
||||
If Not r.IsEmptyString Then
|
||||
NextPage = RegexReplace(r, NextPageRegex)
|
||||
UpdateMediaList(RegexReplace(r, PhotoRegEx), Mode.Internal)
|
||||
UpdateMediaList(RegexReplace(r, PhotoRegExExt), Mode.External)
|
||||
If NextPage = LatestPage Or NextPage.IsEmptyString Then Exit Do Else LatestPage = NextPage
|
||||
Else
|
||||
Exit Do
|
||||
End If
|
||||
Loop
|
||||
|
||||
If _TempMediaList.ListExists And _ContentList.ListExists Then _
|
||||
_TempMediaList.RemoveAll(Function(m) _ContentList.Exists(Function(mm) mm.URL = m.URL))
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub UpdateMediaList(ByVal l As List(Of String), ByVal m As Mode)
|
||||
If l.ListExists Then
|
||||
Dim f As SFile
|
||||
Dim u$
|
||||
Dim exists As Boolean
|
||||
Dim r As RParams
|
||||
Dim ude As New ErrorsDescriber(EDP.ReturnValue)
|
||||
For Each url$ In l
|
||||
If Not url.IsEmptyString Then u = SymbolsConverter.Decode(url, {Converters.HTML, Converters.ASCII}, ude) Else u = String.Empty
|
||||
If Not u.IsEmptyString Then
|
||||
exists = Not IsEmptyString(RegexReplace(u, FileExistsRegEx))
|
||||
If m = Mode.Internal Then
|
||||
r = FileRegEx
|
||||
Else
|
||||
r = FileRegExExt
|
||||
If Not exists Then
|
||||
r = FileRegExExt2
|
||||
exists = Not IsEmptyString(RegexReplace(u, FileRegExExt2))
|
||||
End If
|
||||
End If
|
||||
If exists Then
|
||||
f = CStr(RegexReplace(u, r))
|
||||
f.Path = MyFile.CutPath.PathNoSeparator
|
||||
f.Separator = "\"
|
||||
If f.Extension.IsEmptyString Then f.Extension = "jpg"
|
||||
_TempMediaList.ListAddValue(New UserMedia With {.Type = UTypes.Picture, .URL = url, .File = f}, TempListAddParams)
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
With Responser : .UseWebClient = True : .UseWebClientCookies = True : .ResetError() : End With
|
||||
UseResponserClient = True
|
||||
DownloadContentDefault(Token)
|
||||
End Sub
|
||||
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
|
||||
Optional ByVal EObj As Object = Nothing) As Integer
|
||||
If Responser.StatusCode = Net.HttpStatusCode.ServiceUnavailable Then
|
||||
MyMainLOG = $"{ToStringForLog()}: LPSG not available"
|
||||
Return 1
|
||||
Else
|
||||
Return 0
|
||||
End If
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
Reference in New Issue
Block a user