mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-16 00:32:18 +00:00
b2a9b22478
Plugins IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function Add IDownloadableMedia interface Removed 'Channel' option from all functions and enums ISiteSettings: added GetSingleMediaInstance function ExchangeOptions: removed 'IsChannel' UserMediaTypes: added Audio and AudioPre enums IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums SCrawler Add YouTube standalone downloader Add gallery-dl & yt-dlp support Remove 'UserInfo' requirement from 'ProfilesSaved' Update 'SiteSettingsBase' to use domains and Netscape cookies UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download Rewritten DomainsContainer Create a universal settings form and PSettingsArttribute Gfycat, Imgur: turn these classes into IUserData to download a single object All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist LPSG: fix attachments; update converters and regex Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support Twitter: fixed gif path bug; fixed downloading saved posts PornHub: hide unnecessary errors; photo galleries bug RedGifs: add 'UserAgent' option Added icons to download progress Rename some objects Completely redesigned standalone downloader form and rewritten its environment WebClient2: update to use tokens Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser' UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found Remove temporary 'EncryptCookies' module MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader) PropertyValueHost: update for option forms compatibility SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler UserFinder: remove old 'Merge' const; remove channel option UserInfo: remove channel option
67 lines
3.3 KiB
VB.net
67 lines
3.3 KiB
VB.net
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
|
' This program is free software: you can redistribute it and/or modify
|
|
' it under the terms of the GNU General Public License as published by
|
|
' the Free Software Foundation, either version 3 of the License, or
|
|
' (at your option) any later version.
|
|
'
|
|
' This program is distributed in the hope that it will be useful,
|
|
' but WITHOUT ANY WARRANTY
|
|
Imports System.Net
|
|
Imports PersonalUtilities.Functions.RegularExpressions
|
|
Namespace API.Base
|
|
Friend NotInheritable Class DownDetector
|
|
Private Shared ReadOnly Property Params As New RParams("x:.'([\S]+?)',.y:.(\d+)", -1, Nothing, RegexReturn.List)
|
|
Private Sub New()
|
|
End Sub
|
|
Friend Structure Data : Implements IRegExCreator, IComparable(Of Data)
|
|
Friend [Date] As Date
|
|
Friend Value As Integer
|
|
Private Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray
|
|
If ParamsArray.ListExists Then
|
|
Try : [Date] = Date.Parse(ParamsArray(0)) : Catch : End Try
|
|
If ParamsArray.Length > 1 Then Value = AConvert(Of Integer)(ParamsArray(1), 0)
|
|
End If
|
|
Return Me
|
|
End Function
|
|
Public Overrides Function ToString() As String
|
|
Return $"{AConvert(Of String)([Date], ADateTime.Formats.BaseDateTime, String.Empty)} [{Value}]"
|
|
End Function
|
|
Private Function CompareTo(ByVal Other As Data) As Integer Implements IComparable(Of Data).CompareTo
|
|
Return [Date].CompareTo(Other.Date) * -1
|
|
End Function
|
|
End Structure
|
|
Friend Shared Function GetData(ByVal Site As String) As List(Of Data)
|
|
Try
|
|
Dim l As List(Of Data) = Nothing
|
|
Dim l2 As List(Of Data) = Nothing
|
|
Using w As New WebClient
|
|
Dim r$ = w.DownloadString($"https://downdetector.co.uk/status/{Site}/")
|
|
If Not r.IsEmptyString Then
|
|
l = RegexFields(Of Data)(r, {Params}, {1, 2})
|
|
If l.ListExists(2) Then
|
|
l.Sort()
|
|
l2 = New List(Of Data)
|
|
Dim d As Data
|
|
Dim eDates As New List(Of Date)
|
|
Dim MaxValue As Func(Of Date, Integer) = Function(dd) (From ddd In l Where ddd.Date = dd Select ddd.Value).DefaultIfEmpty(0).Max
|
|
For i% = 0 To l.Count - 1
|
|
If Not eDates.Contains(l(i).Date) Then
|
|
d = l(i)
|
|
d.Value = MaxValue(d.Date)
|
|
l2.Add(d)
|
|
eDates.Add(d.Date)
|
|
End If
|
|
Next
|
|
eDates.Clear()
|
|
l.Clear()
|
|
l2.Sort()
|
|
End If
|
|
End If
|
|
End Using
|
|
Return l2
|
|
Catch ex As Exception
|
|
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"[DownDetector.GetData({Site})]")
|
|
End Try
|
|
End Function
|
|
End Class
|
|
End Namespace |