mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-16 16:52:17 +00:00
2023.4.28.0
Plugins IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function Add IDownloadableMedia interface Removed 'Channel' option from all functions and enums ISiteSettings: added GetSingleMediaInstance function ExchangeOptions: removed 'IsChannel' UserMediaTypes: added Audio and AudioPre enums IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums SCrawler Add YouTube standalone downloader Add gallery-dl & yt-dlp support Remove 'UserInfo' requirement from 'ProfilesSaved' Update 'SiteSettingsBase' to use domains and Netscape cookies UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download Rewritten DomainsContainer Create a universal settings form and PSettingsArttribute Gfycat, Imgur: turn these classes into IUserData to download a single object All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist LPSG: fix attachments; update converters and regex Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support Twitter: fixed gif path bug; fixed downloading saved posts PornHub: hide unnecessary errors; photo galleries bug RedGifs: add 'UserAgent' option Added icons to download progress Rename some objects Completely redesigned standalone downloader form and rewritten its environment WebClient2: update to use tokens Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser' UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found Remove temporary 'EncryptCookies' module MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader) PropertyValueHost: update for option forms compatibility SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler UserFinder: remove old 'Merge' const; remove channel option UserInfo: remove channel option
This commit is contained in:
Andy
@@ -7,12 +7,70 @@
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Net
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.API.YouTube.Objects
|
||||
Imports SCrawler.Plugin.Hosts
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Namespace API.Gfycat
|
||||
Friend NotInheritable Class Envir
|
||||
Private Sub New()
|
||||
Friend NotInheritable Class Envir : Inherits UserDataBase
|
||||
Friend Const SiteKey As String = "AndyProgram_Gfycat"
|
||||
Friend Const SiteName As String = "Gfycat"
|
||||
Friend Sub New()
|
||||
End Sub
|
||||
#Region "UserDataBase Support"
|
||||
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As PersonalUtilities.Functions.XML.XmlFile, ByVal Loading As Boolean)
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
|
||||
SeparateVideoFolder = False
|
||||
DownloadContentDefault(Token)
|
||||
End Sub
|
||||
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
|
||||
Optional ByVal EObj As Object = Nothing) As Integer
|
||||
Return 0
|
||||
End Function
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
Private _IsRedGifs As Boolean = False
|
||||
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
Dim urlVideo$ = GetVideo(Data.URL)
|
||||
If Not urlVideo.IsEmptyString Then
|
||||
If urlVideo.Contains("redgifs.com") Then
|
||||
_IsRedGifs = True
|
||||
DirectCast(Settings(RedGifs.RedGifsSiteKey).Source, RedGifs.SiteSettings).UpdateTokenIfRequired()
|
||||
Dim newData As IYouTubeMediaContainer = Settings(RedGifs.RedGifsSiteKey).GetSingleMediaInstance(urlVideo, Data.File)
|
||||
If Not newData Is Nothing Then
|
||||
newData.Progress = Data.Progress
|
||||
newData.Download(Data.UseCookies, Token)
|
||||
YouTubeMediaContainerBase.Update(newData, Data)
|
||||
DirectCast(Data, DownloadableMediaHost).ExchangeData(newData, Data)
|
||||
With DirectCast(Data, YouTubeMediaContainerBase)
|
||||
.Site = RedGifs.RedGifsSite
|
||||
.SiteKey = RedGifs.RedGifsSiteKey
|
||||
.SiteIcon = Settings(RedGifs.RedGifsSiteKey).Source.Image
|
||||
End With
|
||||
Else
|
||||
Throw New Exception($"Unable to get RedGifs instance{vbCr}{Data.URL}{vbCr}{urlVideo}")
|
||||
End If
|
||||
Else
|
||||
Dim m As New UserMedia(urlVideo, UserMedia.Types.Video) With {.URL_BASE = Data.URL}
|
||||
m.File.Path = Data.File.Path
|
||||
_TempMediaList.Add(m)
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadSingleObject_CreateMedia(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
If Not _IsRedGifs Then MyBase.DownloadSingleObject_CreateMedia(Data, Token)
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadSingleObject_PostProcessing(ByVal Data As IYouTubeMediaContainer, Optional ByVal ResetTitle As Boolean = True)
|
||||
If Not _IsRedGifs Then MyBase.DownloadSingleObject_PostProcessing(Data, ResetTitle)
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadSingleObject_Download(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
If Not _IsRedGifs Then MyBase.DownloadSingleObject_Download(Data, Token)
|
||||
End Sub
|
||||
#End Region
|
||||
Friend Shared Function GetVideo(ByVal URL As String) As String
|
||||
Try
|
||||
Dim r$
|
||||
@@ -33,14 +91,22 @@ Namespace API.Gfycat
|
||||
Dim e As EDP = EDP.ReturnValue
|
||||
If TypeOf ex Is WebException Then
|
||||
Dim obj As HttpWebResponse = TryCast(DirectCast(ex, WebException).Response, HttpWebResponse)
|
||||
If Not If(obj?.StatusCode, HttpStatusCode.OK) = HttpStatusCode.NotFound Then e += EDP.SendInLog
|
||||
If Not If(obj?.StatusCode, HttpStatusCode.OK) = HttpStatusCode.NotFound Then e += EDP.SendToLog
|
||||
End If
|
||||
Return ErrorsDescriber.Execute(e, ex, $"[API.Gfycat.Envir.GetVideo({URL})]", String.Empty)
|
||||
End Try
|
||||
End Function
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String) As IEnumerable(Of UserMedia)
|
||||
Dim u$ = GetVideo(URL)
|
||||
Return If(u.IsEmptyString, Nothing, {New UserMedia(u, UserMedia.Types.Video)})
|
||||
Friend Shared Function GetSingleMediaInstance(ByVal URL As String, ByVal OutputFile As SFile) As IYouTubeMediaContainer
|
||||
If Not URL.IsEmptyString AndAlso URL.Contains("gfycat") Then
|
||||
Return New DownloadableMediaHost(URL, OutputFile) With {
|
||||
.Instance = New Envir,
|
||||
.Site = SiteName,
|
||||
.SiteKey = SiteKey,
|
||||
.SiteIcon = Nothing
|
||||
}
|
||||
Else
|
||||
Return Nothing
|
||||
End If
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
Reference in New Issue
Block a user