diff --git a/spec/invidious/channel/community_spec.cr b/spec/invidious/channel/community_spec.cr new file mode 100644 index 000000000..0639b2468 --- /dev/null +++ b/spec/invidious/channel/community_spec.cr @@ -0,0 +1,23 @@ +require "../../spec_helper" + +Spectator.describe Invidious::Channels::Community do + describe ".produce_channel_community_continuation" do + it "correctly produces a continuation token for a channel community" do + expect(Invidious::Channels::Community.produce_channel_community_continuation("UCCj956IF62FbT7Gouszaj9w", "Egljb21tdW5pdHm4")).to eq("4qmFsgIsEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaEEVnbGpiMjF0ZFc1cGRIbTQ%3D") + expect(Invidious::Channels::Community.produce_channel_community_continuation("UCCj956IF62FbT7Gouszaj9w", "Egljb21tdW5pdHm4AQCqAyQaIBIaVWd3cE9NQmVwWEdjclhsUHg2WjRBYUFCQ1FIZGgDKAA%3D")).to eq("4qmFsgJmEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaSkVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2QzY0U5TlFtVndXRWRqY2xoc1VIZzJXalJCWVVGQ1ExRklaR2dES0FBJTNE") + + expect(Invidious::Channels::Community.produce_channel_community_continuation("UC-lHJZR3Gqxm24_Vd_AJ5Yw", "Egljb21tdW5pdHm4AQCqAyQaIBIaVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1FIZGgDKAA%3D")).to eq("4qmFsgJmEhhVQy1sSEpaUjNHcXhtMjRfVmRfQUo1WXcaSkVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2Q1UlRJMk5XMXJVa2syY0U5dVMyMW5iRFJCWVVGQ1ExRklaR2dES0FBJTNE") + expect(Invidious::Channels::Community.produce_channel_community_continuation("UC-lHJZR3Gqxm24_Vd_AJ5Yw", "Egljb21tdW5pdHm4AQCqA-cOCsAOUVVSVFNsOXBNWEYxYlVablFXaGFiWFJNTW5WM1ZHSXdPVU5EWTNoeFJWWlVjRWRGVTBOa1prTktjVUoyWjBZemNEZHRPV2cwV1hWbVJtaFVPWFJwVjJaUU4xTXlNRWRaYlZwSVFUa3dlak5pTUV0dll6QkRVMlpsWHpoVFdUbHFSR0o1YkRkM1kydEhMVTVwWDFCdFdXOUhjR0Z6ZEMxbldVcEhUMjkzUm1saGRXSkViVmR6ZFhwd1QxTnpOME54TW5KUloxQlBkME5QU1VWVWMybHlNbFZvUVV0NlVIZFVhMVV5UzNWUmJHRldkRmszU1dKd1pVUllVMkZFVG1aV1ZsRnVUMGhsZFd0T01sVndTbGd3TkhweVdDMVBTRUphV25GNk5Yb3dYMWRCVTFnMlltODBPRmhIV205WlQwNW1YMjV1UlVKTWNucHNNSGR5Y1hKaFltUkVkblJYZG1Kc1FVaHFUV3BwTkc5R1pUQkVlbGw2ZHpSM2FISlBTSFJoYjJGbVMwNTBiV1pxV2pCSVNWWnZTalpRT0RoclVGVmhia1p5VFhsaWFTMVBjREZZV1dSTFdERkZjSHB0ZUhseWFtRXdNR1JmTkhOWmFEVlZTbVZ1ZUVkRU1XRlFhbU4xVERabk4wdDVSSGxHU2xsT1VEQlJXR1ZLTUhGM1UwWkJTSE5oWkRWQ2NXZHNaMFpqYW1ST1YxZFlhMDVOVUZSSFZWVktRekZSYVhodlUxTm1SV1EwTUdsdWNEWXlPV1YwUjNkcGFVcEVTM040YUZadmRXbHJhblkyZFdFelNHWXpUV3hMYURCa2JIRTFSblJ4Wms4NU1XbGtOM0pHYjBGeU4xZFJNMU5qYkZCd05rZE9jV1JqT1hGRGIyNU5Xak5TUlhkemFsUXRObGt4UWxkUE16ZGFaRTlxVGtaZlIweEhRbXRNWXpCWE9GUjNOMHBsYVhwS2RtSlZkMmxGTVhCbVNIWkdkVTFJY0MxbFdYSkVZM0V0ZFROWWRtVlFlV3hhYlVKMmVreGZUMGxOU2xaSlRFTlBZMVpEUjFwd1RHZFhZMmhIYVVKakxUSmFabXd0U1RNeFJEWkhlSGhYTkhOMU1GZGhOMjFCVlVnNGNFTlJXSGx2WW5ScWNUaHZXWGxKT1d0TVRXc3lRMWc0Um5wU2JEVjBlRGxpTW5vMVRYaEtkelExY201S1JHSmZkamhmTlhOWmRGYzRjak5FVVdkMlpXVnNRWEJyZW5OdFpHcEljVGhWYzFsZkxWa3dRVTkyTVZVMmIyMTNVeTFLVEUxeFIwUldRbmc0VEdsTlpGVktjVmxzTkZGa1UwazFabE0wZUhsRk5WZ3lWR0ZaYzJadlYyaHRPRFpzTjNCT1dHRnBiMHhUVDBkMmRuZFVOMlptVm05dWIwRTFZVkZuYldKNmIwMUNaMng2VGkxSk56bHhXV3BJVGt4RFYwVllUM05pTVcwemRHc3lUVWN6TVVKcVRHdElNVWg1YmtKQmVrbFNVMnczZEVKUlJGOUlNVWRyZERsbFJraHVYekJXZUhGbE1rTTBlVE40YVU1T1pFcGpVMkpFZFMxWVdITjNTMnhWVjJwYVgzVXRXbGcwZG5OSE1qUXpYMlJHTVhSV1kxWkZRMlZwU25OdVlXTkdVek5wVUd4b2FUbDVSRVp4YVhsbFRqbG1aRWxYVFZCMVFWbG9OMEl3TW5KV1JUVjRkREJLZG5obmJGZHhSVlY1ZWpjMFIyeGlZemRIVmkxeFpESmlaMnhFZGxkcVRuSjZNVEZWUkRWamVIQlFkRk5DVmtSU2RITlRaSGhWZG05WE9VUkNhWEYwTm1kSFRtb3RNV1pNYlhSeVJWTnJhRWhIVDB0SU0yVkxUbFZ2V1VGNlJTMDJialJZYkRKdFFUVnJhRVJ4WmpjeFptcERNR001UmpkM2QwNW1VRXd5YUZCZlEwWjFSbEUzY0doRk5ISkZZMWxTTWs5d2RXRnhiRzFrYjBVMmIxWkJaRzkyU2xneFZWOXNiMDVWWkUxRFJ6QjBjWGhpVjBVMldYY3pTUzF4UVcxa1RuZEJRVGRvWVZFNGNsSTBaVUl0UmxacVdETnJXazVLY21aRk9HVndRbWxqUjB0blRFZEZVR3N6YzJOclkwSTNlVlZZVEdkcE1YQkdiMHAyZVU1aGRVZFdVblJQYVhaQlZtdHZSa0UzTFU1Sk1XaFJRMUpMV2kxSWJ6WkxjWEkxZGtSTWJsOVdUa0ZFVmpKZmMwUlFWV3gwUTJ0TFRsbDJaM2gxZFVOSVkzbEVORUpRZVUxMVREQnpOMVowWDI1MWRrVmlUMU54TkRkUk5rVjViMEpRTUZGNmR6RlJSR2RxY1U1eVgwNTBjMDkxWm14R2NUVjBlRkJGT1dGVmFXeFJTMEZYYldwQlVVbHNOVmgwZERZdGFFRlViMWxmUjFWc1EycG1WVkJQV0hkcFVRPT0aIBIaVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1FIZGgDKGM%3D")).to eq("4qmFsgKXFBIYVUMtbEhKWlIzR3F4bTI0X1ZkX0FKNVl3GvoTRWdsamIyMXRkVzVwZEhtNEFRQ3FBLWNPQ3NBT1VWVlNWRk5zT1hCTldFWXhZbFZhYmxGWGFHRmlXRkpOVFc1V00xWkhTWGRQVlU1RVdUTm9lRkpXV2xWalJXUkdWVEJPYTFwclRrdGpWVW95V2pCWmVtTkVaSFJQVjJjd1YxaFdiVkp0YUZWUFdGSndWakphVVU0eFRYbE5SV1JhWWxad1NWRlVhM2RsYWs1cFRVVjBkbGw2UWtSVk1scHNXSHBvVkZkVWJIRlNSMG8xWWtSa00xa3lkRWhNVlRWd1dERkNkRmRYT1VoalIwWjZaRU14YmxkVmNFaFVNamt6VW0xc2FHUlhTa1ZpVm1SNlpGaHdkMVF4VG5wT01FNTRUVzVLVWxveFFsQmtNRTVRVTFWV1ZXTXliSGxOYkZadlVWVjBObFZJWkZWaE1WVjVVek5XVW1KSFJsZGtSbXN6VTFkS2QxcFZVbGxWTWtaRlZHMWFWMVpzUm5WVU1HaHNaRmQwVDAxc1ZuZFRiR2QzVGtod2VWZERNVkJUUlVwaFYyNUdOazVZYjNkWU1XUkNWVEZuTWxsdE9EQlBSbWhJVjIwNVdsUXdOVzFZTWpWMVVsVktUV051Y0hOTlNHUjVZMWhLYUZsdFVrVmtibEpZWkcxS2MxRlZhSEZVVjNCd1RrYzVSMXBVUWtWbGJHdzJaSHBTTTJGSVNsQlRTRkpvWWpKR2JWTXdOVEJpVjFweFYycENTVk5XV25aVGFscFJUMFJvY2xWR1ZtaGlhMXA1VkZoc2FXRlRNVkJqUkVaWlYxZFNURmRFUmtaalNIQjBaVWhzZVdGdFJYZE5SMUptVGtoT1dtRkVWbFpUYlZaMVpVVmtSVTFYUmxGaGJVNHhWRVJhYms0d2REVlNTR3hIVTJ4c1QxVkVRbEpYUjFaTFRVaEdNMVV3V2tKVFNFNW9Xa1JXUTJOWFpITmFNRnBxWVcxU1QxWXhaRmxoTURWT1ZVWlNTRlpXVmt0UmVrWlNZVmhvZGxVeFRtMVNWMUV3VFVkc2RXTkVXWGxQVjFZd1VqTmtjR0ZWY0VWVE0wNDBZVVphZG1SWGJISmhibGt5WkZkRmVsTkhXWHBVVjNoTVlVUkNhMkpJUlRGU2JsSjRXbXM0TlUxWGJHdE9NMHBIWWpCR2VVNHhaRkpOTVU1cVlrWkNkMDVyWkU5alYxSnFUMWhHUkdJeU5VNVhhazVUVWxoa2VtRnNVWFJPYkd0NFVXeGtVRTE2WkdGYVJUbHhWR3RhWmxJd2VFaFJiWFJOV1hwQ1dFOUdVak5PTUhCc1lWaHdTMlJ0U2xaa01teEdUVmhDYlZOSVdrZGtWVEZKWTBNeGJGZFlTa1ZaTTBWMFpGUk9XV1J0VmxGbFYzaGhZbFZLTW1WcmVHWlVNR3hPVTJ4YVNsUkZUbEJaTVZwRVVqRndkMVJIWkZoWk1taElZVlZLYWt4VVNtRmFiWGQwVTFSTmVGSkVXa2hsU0doWVRraE9NVTFHWkdoT01qRkNWbFZuTkdORlRsSlhTR3gyV1c1U2NXTlVhSFpYV0d4S1QxZDBUVlJYYzNsUk1XYzBVbTV3VTJKRVZqQmxSR3hwVFc1dk1WUllhRXRrZWxFeFkyMDFTMUpIU21aa2FtaG1UbGhPV21SR1l6UmphazVGVlZka01scFhWbk5SV0VKeVpXNU9kRnBIY0VsalZHaFdZekZzWmt4V2EzZFJWVGt5VFZaVk1tSXlNVE5WZVRGTFZFVXhlRkl3VWxkUmJtYzBWRWRzVGxwR1ZrdGpWbXh6VGtaR2ExVXdhekZhYkUwd1pVaHNSazVXWjNsV1IwWmFZekphZGxZeWFIUlBSRnB6VGpOQ1QxZEhSbkJpTUhoVVZEQmtNbVJ1WkZWT01scHRWbTA1ZFdJd1JURlpWa1p1WWxkS05tSXdNVU5hTW5nMlZHa3hTazU2YkhoWFYzQkpWR3Q0UkZZd1ZsbFVNMDVwVFZjd2VtUkhjM2xVVldONlRWVktjVlJIZEVsTlZXZzFZbXRLUW1WcmJGTlZNbmN6WkVWS1VsSkdPVWxOVldSeVpFUnNiRkpyYUhWWWVrSlhaVWhHYkUxclRUQmxWRTQwWVZVMVQxcEZjR3BWTWtwRlpGTXhXVmRJVGpOVE1uaFdWakp3WVZnelZYUlhiR2N3Wkc1T1NFMXFVWHBZTWxKSFRWaFNWMWt4V2taUk1sWndVMjVPZFZsWFRrZFZlazV3VlVkNGIyRlViRFZTUlZwNFlWaHNiRlJxYkcxYVJXeFlWRlpDTVZGV2JHOU9NRWwzVFc1S1YxSlVWalJrUkVKTFpHNW9ibUpHWkhoU1ZsWTFaV3BqTUZJeWVHbFplbVJJVm1reGVGcEVTbWxhTW5oRlpHeGtjVlJ1U2paTlZFWldVa1JXYW1WSVFsRmtSazVEVm10U1UyUklUbFJhU0doV1pHMDVXRTlWVWtOaFdFWXdUbTFrU0ZSdGIzUk5WMXBOWWxoU2VWSldUbkpoUldoSVZEQjBTVTB5Vmt4VWJGWjJWMVZHTmxKVE1ESmlhbEpaWWtSS2RGRlVWbkpoUlZKNFdtcGplRnB0Y0VSTlIwMDFVbXBrTTJRd05XMVZSWGQ1WVVaQ1psRXdXakZTYkVVelkwZG9SazVJU2taWk1XeFRUV3M1ZDJSWFJuaGlSekZyWWpCVk1tSXhXa0phUnpreVUyeG5lRlpXT1hOaU1EVldXa1V4UkZKNlFqQmpXR2hwVmpCVk1sZFlZM3BUVXpGNFVWY3hhMVJ1WkVKUlZHUnZXVlpGTkdOc1NUQmFWVWwwVW14YWNWZEVUbkpYYXpWTFkyMWFSazlIVm5kUmJXeHFVakIwYmxSRlpFWlZSM042WXpKT2Nsa3dTVE5sVmxaWlZFZGtjRTFZUWtkaU1IQXlaVlUxYUdSVlpGZFZibEpRWVZoYVFsWnRkSFpTYTBVelRGVTFTazFYYUZKUk1VcE1WMmt4U1dKNldreGpXRWt4Wkd0U1RXSnNPVmRVYTBaRlZtcEtabU13VWxGV1YzZ3dVVEowVEZSc2JESmFNMmd4WkZWT1NWa3piRVZPUlVwUlpWVXhNVlJFUW5wT01Wb3dXREkxTVdSclZtbFVNVTU0VGtSa1VrNXJWalZpTUVwUlRVWkdObVI2UmxKU1IyUnhZMVUxZVZnd05UQmpNRGt4V20xNFIyTlVWakJsUmtKR1QxZEdWbUZYZUZKVE1FWllZbGR3UWxWVmJITk9WbWd3WkVSWmRHRkZSbFZpTVd4bVVqRldjMUV5Y0cxV1ZrSlFWMGhrY0ZWUlBUMGFJQklhVldkNVJUSTJOVzFyVWtrMmNFOXVTMjFuYkRSQllVRkNRMUZJWkdnREtHTSUzRA%3D%3D") + end + end + + describe ".extract_channel_community_cursor" do + it "correctly extracts a community cursor from a given continuation" do + expect(Invidious::Channels::Community.extract_channel_community_cursor("4qmFsgIsEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaEEVnbGpiMjF0ZFc1cGRIbTQ%3D")).to eq("Egljb21tdW5pdHk=") + expect(Invidious::Channels::Community.extract_channel_community_cursor("4qmFsgJoEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaTEVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2QzY0U5TlFtVndXRWRqY2xoc1VIZzJXalJCWVVGQ1ExRklaR2dES0FBJTI1M0Q%3D")).to eq("Egljb21tdW5pdHm4AQCqAyQaIEhkaAMSGlVnd3BPTUJlcFhHY3JYbFB4Nlo0QWFBQkNRKAA=") + + expect(Invidious::Channels::Community.extract_channel_community_cursor("4qmFsgJoEhhVQy1sSEpaUjNHcXhtMjRfVmRfQUo1WXcaTEVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2Q1UlRJMk5XMXJVa2syY0U5dVMyMW5iRFJCWVVGQ1ExRklaR2dES0FBJTI1M0Q%3D")).to eq("Egljb21tdW5pdHm4AQCqAyQaIEhkaAMSGlVneUUyNjVta1JJNnBPbkttZ2w0QWFBQkNRKAA=") + expect(Invidious::Channels::Community.extract_channel_community_cursor("4qmFsgKZFBIYVUMtbEhKWlIzR3F4bTI0X1ZkX0FKNVl3GvwTRWdsamIyMXRkVzVwZEhtNEFRQ3FBLWNPQ3NBT1VWVlNWRk5zT1hCTldFWXhZbFZhYmxGWGFHRmlXRkpOVFc1V00xWkhTWGRQVlU1RVdUTm9lRkpXV2xWalJXUkdWVEJPYTFwclRrdGpWVW95V2pCWmVtTkVaSFJQVjJjd1YxaFdiVkp0YUZWUFdGSndWakphVVU0eFRYbE5SV1JhWWxad1NWRlVhM2RsYWs1cFRVVjBkbGw2UWtSVk1scHNXSHBvVkZkVWJIRlNSMG8xWWtSa00xa3lkRWhNVlRWd1dERkNkRmRYT1VoalIwWjZaRU14YmxkVmNFaFVNamt6VW0xc2FHUlhTa1ZpVm1SNlpGaHdkMVF4VG5wT01FNTRUVzVLVWxveFFsQmtNRTVRVTFWV1ZXTXliSGxOYkZadlVWVjBObFZJWkZWaE1WVjVVek5XVW1KSFJsZGtSbXN6VTFkS2QxcFZVbGxWTWtaRlZHMWFWMVpzUm5WVU1HaHNaRmQwVDAxc1ZuZFRiR2QzVGtod2VWZERNVkJUUlVwaFYyNUdOazVZYjNkWU1XUkNWVEZuTWxsdE9EQlBSbWhJVjIwNVdsUXdOVzFZTWpWMVVsVktUV051Y0hOTlNHUjVZMWhLYUZsdFVrVmtibEpZWkcxS2MxRlZhSEZVVjNCd1RrYzVSMXBVUWtWbGJHdzJaSHBTTTJGSVNsQlRTRkpvWWpKR2JWTXdOVEJpVjFweFYycENTVk5XV25aVGFscFJUMFJvY2xWR1ZtaGlhMXA1VkZoc2FXRlRNVkJqUkVaWlYxZFNURmRFUmtaalNIQjBaVWhzZVdGdFJYZE5SMUptVGtoT1dtRkVWbFpUYlZaMVpVVmtSVTFYUmxGaGJVNHhWRVJhYms0d2REVlNTR3hIVTJ4c1QxVkVRbEpYUjFaTFRVaEdNMVV3V2tKVFNFNW9Xa1JXUTJOWFpITmFNRnBxWVcxU1QxWXhaRmxoTURWT1ZVWlNTRlpXVmt0UmVrWlNZVmhvZGxVeFRtMVNWMUV3VFVkc2RXTkVXWGxQVjFZd1VqTmtjR0ZWY0VWVE0wNDBZVVphZG1SWGJISmhibGt5WkZkRmVsTkhXWHBVVjNoTVlVUkNhMkpJUlRGU2JsSjRXbXM0TlUxWGJHdE9NMHBIWWpCR2VVNHhaRkpOTVU1cVlrWkNkMDVyWkU5alYxSnFUMWhHUkdJeU5VNVhhazVUVWxoa2VtRnNVWFJPYkd0NFVXeGtVRTE2WkdGYVJUbHhWR3RhWmxJd2VFaFJiWFJOV1hwQ1dFOUdVak5PTUhCc1lWaHdTMlJ0U2xaa01teEdUVmhDYlZOSVdrZGtWVEZKWTBNeGJGZFlTa1ZaTTBWMFpGUk9XV1J0VmxGbFYzaGhZbFZLTW1WcmVHWlVNR3hPVTJ4YVNsUkZUbEJaTVZwRVVqRndkMVJIWkZoWk1taElZVlZLYWt4VVNtRmFiWGQwVTFSTmVGSkVXa2hsU0doWVRraE9NVTFHWkdoT01qRkNWbFZuTkdORlRsSlhTR3gyV1c1U2NXTlVhSFpYV0d4S1QxZDBUVlJYYzNsUk1XYzBVbTV3VTJKRVZqQmxSR3hwVFc1dk1WUllhRXRrZWxFeFkyMDFTMUpIU21aa2FtaG1UbGhPV21SR1l6UmphazVGVlZka01scFhWbk5SV0VKeVpXNU9kRnBIY0VsalZHaFdZekZzWmt4V2EzZFJWVGt5VFZaVk1tSXlNVE5WZVRGTFZFVXhlRkl3VWxkUmJtYzBWRWRzVGxwR1ZrdGpWbXh6VGtaR2ExVXdhekZhYkUwd1pVaHNSazVXWjNsV1IwWmFZekphZGxZeWFIUlBSRnB6VGpOQ1QxZEhSbkJpTUhoVVZEQmtNbVJ1WkZWT01scHRWbTA1ZFdJd1JURlpWa1p1WWxkS05tSXdNVU5hTW5nMlZHa3hTazU2YkhoWFYzQkpWR3Q0UkZZd1ZsbFVNMDVwVFZjd2VtUkhjM2xVVldONlRWVktjVlJIZEVsTlZXZzFZbXRLUW1WcmJGTlZNbmN6WkVWS1VsSkdPVWxOVldSeVpFUnNiRkpyYUhWWWVrSlhaVWhHYkUxclRUQmxWRTQwWVZVMVQxcEZjR3BWTWtwRlpGTXhXVmRJVGpOVE1uaFdWakp3WVZnelZYUlhiR2N3Wkc1T1NFMXFVWHBZTWxKSFRWaFNWMWt4V2taUk1sWndVMjVPZFZsWFRrZFZlazV3VlVkNGIyRlViRFZTUlZwNFlWaHNiRlJxYkcxYVJXeFlWRlpDTVZGV2JHOU9NRWwzVFc1S1YxSlVWalJrUkVKTFpHNW9ibUpHWkhoU1ZsWTFaV3BqTUZJeWVHbFplbVJJVm1reGVGcEVTbWxhTW5oRlpHeGtjVlJ1U2paTlZFWldVa1JXYW1WSVFsRmtSazVEVm10U1UyUklUbFJhU0doV1pHMDVXRTlWVWtOaFdFWXdUbTFrU0ZSdGIzUk5WMXBOWWxoU2VWSldUbkpoUldoSVZEQjBTVTB5Vmt4VWJGWjJWMVZHTmxKVE1ESmlhbEpaWWtSS2RGRlVWbkpoUlZKNFdtcGplRnB0Y0VSTlIwMDFVbXBrTTJRd05XMVZSWGQ1WVVaQ1psRXdXakZTYkVVelkwZG9SazVJU2taWk1XeFRUV3M1ZDJSWFJuaGlSekZyWWpCVk1tSXhXa0phUnpreVUyeG5lRlpXT1hOaU1EVldXa1V4UkZKNlFqQmpXR2hwVmpCVk1sZFlZM3BUVXpGNFVWY3hhMVJ1WkVKUlZHUnZXVlpGTkdOc1NUQmFWVWwwVW14YWNWZEVUbkpYYXpWTFkyMWFSazlIVm5kUmJXeHFVakIwYmxSRlpFWlZSM042WXpKT2Nsa3dTVE5sVmxaWlZFZGtjRTFZUWtkaU1IQXlaVlUxYUdSVlpGZFZibEpRWVZoYVFsWnRkSFpTYTBVelRGVTFTazFYYUZKUk1VcE1WMmt4U1dKNldreGpXRWt4Wkd0U1RXSnNPVmRVYTBaRlZtcEtabU13VWxGV1YzZ3dVVEowVEZSc2JESmFNMmd4WkZWT1NWa3piRVZPUlVwUlpWVXhNVlJFUW5wT01Wb3dXREkxTVdSclZtbFVNVTU0VGtSa1VrNXJWalZpTUVwUlRVWkdObVI2UmxKU1IyUnhZMVUxZVZnd05UQmpNRGt4V20xNFIyTlVWakJsUmtKR1QxZEdWbUZYZUZKVE1FWllZbGR3UWxWVmJITk9WbWd3WkVSWmRHRkZSbFZpTVd4bVVqRldjMUV5Y0cxV1ZrSlFWMGhrY0ZWUlBUMGFJQklhVldkNVJUSTJOVzFyVWtrMmNFOXVTMjFuYkRSQllVRkNRMUZJWkdnREtHTSUyNTNE")).to eq("Egljb21tdW5pdHm4AQCqA-kOCsAOUVVSVFNsOXBNWEYxYlVablFXaGFiWFJNTW5WM1ZHSXdPVU5EWTNoeFJWWlVjRWRGVTBOa1prTktjVUoyWjBZemNEZHRPV2cwV1hWbVJtaFVPWFJwVjJaUU4xTXlNRWRaYlZwSVFUa3dlak5pTUV0dll6QkRVMlpsWHpoVFdUbHFSR0o1YkRkM1kydEhMVTVwWDFCdFdXOUhjR0Z6ZEMxbldVcEhUMjkzUm1saGRXSkViVmR6ZFhwd1QxTnpOME54TW5KUloxQlBkME5QU1VWVWMybHlNbFZvUVV0NlVIZFVhMVV5UzNWUmJHRldkRmszU1dKd1pVUllVMkZFVG1aV1ZsRnVUMGhsZFd0T01sVndTbGd3TkhweVdDMVBTRUphV25GNk5Yb3dYMWRCVTFnMlltODBPRmhIV205WlQwNW1YMjV1UlVKTWNucHNNSGR5Y1hKaFltUkVkblJYZG1Kc1FVaHFUV3BwTkc5R1pUQkVlbGw2ZHpSM2FISlBTSFJoYjJGbVMwNTBiV1pxV2pCSVNWWnZTalpRT0RoclVGVmhia1p5VFhsaWFTMVBjREZZV1dSTFdERkZjSHB0ZUhseWFtRXdNR1JmTkhOWmFEVlZTbVZ1ZUVkRU1XRlFhbU4xVERabk4wdDVSSGxHU2xsT1VEQlJXR1ZLTUhGM1UwWkJTSE5oWkRWQ2NXZHNaMFpqYW1ST1YxZFlhMDVOVUZSSFZWVktRekZSYVhodlUxTm1SV1EwTUdsdWNEWXlPV1YwUjNkcGFVcEVTM040YUZadmRXbHJhblkyZFdFelNHWXpUV3hMYURCa2JIRTFSblJ4Wms4NU1XbGtOM0pHYjBGeU4xZFJNMU5qYkZCd05rZE9jV1JqT1hGRGIyNU5Xak5TUlhkemFsUXRObGt4UWxkUE16ZGFaRTlxVGtaZlIweEhRbXRNWXpCWE9GUjNOMHBsYVhwS2RtSlZkMmxGTVhCbVNIWkdkVTFJY0MxbFdYSkVZM0V0ZFROWWRtVlFlV3hhYlVKMmVreGZUMGxOU2xaSlRFTlBZMVpEUjFwd1RHZFhZMmhIYVVKakxUSmFabXd0U1RNeFJEWkhlSGhYTkhOMU1GZGhOMjFCVlVnNGNFTlJXSGx2WW5ScWNUaHZXWGxKT1d0TVRXc3lRMWc0Um5wU2JEVjBlRGxpTW5vMVRYaEtkelExY201S1JHSmZkamhmTlhOWmRGYzRjak5FVVdkMlpXVnNRWEJyZW5OdFpHcEljVGhWYzFsZkxWa3dRVTkyTVZVMmIyMTNVeTFLVEUxeFIwUldRbmc0VEdsTlpGVktjVmxzTkZGa1UwazFabE0wZUhsRk5WZ3lWR0ZaYzJadlYyaHRPRFpzTjNCT1dHRnBiMHhUVDBkMmRuZFVOMlptVm05dWIwRTFZVkZuYldKNmIwMUNaMng2VGkxSk56bHhXV3BJVGt4RFYwVllUM05pTVcwemRHc3lUVWN6TVVKcVRHdElNVWg1YmtKQmVrbFNVMnczZEVKUlJGOUlNVWRyZERsbFJraHVYekJXZUhGbE1rTTBlVE40YVU1T1pFcGpVMkpFZFMxWVdITjNTMnhWVjJwYVgzVXRXbGcwZG5OSE1qUXpYMlJHTVhSV1kxWkZRMlZwU25OdVlXTkdVek5wVUd4b2FUbDVSRVp4YVhsbFRqbG1aRWxYVFZCMVFWbG9OMEl3TW5KV1JUVjRkREJLZG5obmJGZHhSVlY1ZWpjMFIyeGlZemRIVmkxeFpESmlaMnhFZGxkcVRuSjZNVEZWUkRWamVIQlFkRk5DVmtSU2RITlRaSGhWZG05WE9VUkNhWEYwTm1kSFRtb3RNV1pNYlhSeVJWTnJhRWhIVDB0SU0yVkxUbFZ2V1VGNlJTMDJialJZYkRKdFFUVnJhRVJ4WmpjeFptcERNR001UmpkM2QwNW1VRXd5YUZCZlEwWjFSbEUzY0doRk5ISkZZMWxTTWs5d2RXRnhiRzFrYjBVMmIxWkJaRzkyU2xneFZWOXNiMDVWWkUxRFJ6QjBjWGhpVjBVMldYY3pTUzF4UVcxa1RuZEJRVGRvWVZFNGNsSTBaVUl0UmxacVdETnJXazVLY21aRk9HVndRbWxqUjB0blRFZEZVR3N6YzJOclkwSTNlVlZZVEdkcE1YQkdiMHAyZVU1aGRVZFdVblJQYVhaQlZtdHZSa0UzTFU1Sk1XaFJRMUpMV2kxSWJ6WkxjWEkxZGtSTWJsOVdUa0ZFVmpKZmMwUlFWV3gwUTJ0TFRsbDJaM2gxZFVOSVkzbEVORUpRZVUxMVREQnpOMVowWDI1MWRrVmlUMU54TkRkUk5rVjViMEpRTUZGNmR6RlJSR2RxY1U1eVgwNTBjMDkxWm14R2NUVjBlRkJGT1dGVmFXeFJTMEZYYldwQlVVbHNOVmgwZERZdGFFRlViMWxmUjFWc1EycG1WVkJQV0hkcFVRPT0aIhIcVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1E9PUhkaAMoYw==") + end + end +end diff --git a/spec/invidious/helpers/tokens_spec.cr b/spec/invidious/helpers/tokens_spec.cr new file mode 100644 index 000000000..2bb47f060 --- /dev/null +++ b/spec/invidious/helpers/tokens_spec.cr @@ -0,0 +1,26 @@ +require "../../spec_helper" + +Spectator.describe Invidious::Helpers::Tokens do + describe ".sign_token" do + it "correctly signs a given hash" do + token = { + "session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "expires" => 1554680038, + "scopes" => [ + ":notifications", + ":subscriptions/*", + "GET:tokens*", + ], + "signature" => "f__2hS20th8pALF305PJFK-D2aVtvefNnQheILHD2vU=", + } + expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"]) + + token = { + "session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "scopes" => [":notifications", "POST:subscriptions/*"], + "signature" => "fNvXoT0MRAL9eE6lTE33CEg8HitYJDOL9a22rSN2Ihg=", + } + expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"]) + end + end +end diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr deleted file mode 100644 index 9fbb6d6fe..000000000 --- a/spec/invidious/helpers_spec.cr +++ /dev/null @@ -1,56 +0,0 @@ -require "../spec_helper" - -CONFIG = Config.from_yaml(File.open("config/config.example.yml")) - -Spectator.describe "Helper" do - describe "#produce_channel_search_continuation" do - it "correctly produces token for searching a specific channel" do - expect(produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") - - expect(produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "По ожиशुपतिरपि子而時ஸ்றீனி", 0)).to eq("4qmFsgKoARIYVUNYdXFTQmxIQUU2WHcteWVKQTBUdW53GiBFZ1p6WldGeVkyZ3dBVGdCWUFGNkJFZEJRVDI0QVFBPVo-0J_QviDQvtC20LjgpLbgpYHgpKrgpKTgpL_gpLDgpKrgpL_lrZDogIzmmYLgrrjgr43grrHgr4Dgrqngrr-aAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") - end - end - - describe "#produce_channel_community_continuation" do - it "correctly produces a continuation token for a channel community" do - expect(produce_channel_community_continuation("UCCj956IF62FbT7Gouszaj9w", "Egljb21tdW5pdHm4")).to eq("4qmFsgIsEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaEEVnbGpiMjF0ZFc1cGRIbTQ%3D") - expect(produce_channel_community_continuation("UCCj956IF62FbT7Gouszaj9w", "Egljb21tdW5pdHm4AQCqAyQaIBIaVWd3cE9NQmVwWEdjclhsUHg2WjRBYUFCQ1FIZGgDKAA%3D")).to eq("4qmFsgJmEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaSkVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2QzY0U5TlFtVndXRWRqY2xoc1VIZzJXalJCWVVGQ1ExRklaR2dES0FBJTNE") - - expect(produce_channel_community_continuation("UC-lHJZR3Gqxm24_Vd_AJ5Yw", "Egljb21tdW5pdHm4AQCqAyQaIBIaVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1FIZGgDKAA%3D")).to eq("4qmFsgJmEhhVQy1sSEpaUjNHcXhtMjRfVmRfQUo1WXcaSkVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2Q1UlRJMk5XMXJVa2syY0U5dVMyMW5iRFJCWVVGQ1ExRklaR2dES0FBJTNE") - expect(produce_channel_community_continuation("UC-lHJZR3Gqxm24_Vd_AJ5Yw", "Egljb21tdW5pdHm4AQCqA-cOCsAOUVVSVFNsOXBNWEYxYlVablFXaGFiWFJNTW5WM1ZHSXdPVU5EWTNoeFJWWlVjRWRGVTBOa1prTktjVUoyWjBZemNEZHRPV2cwV1hWbVJtaFVPWFJwVjJaUU4xTXlNRWRaYlZwSVFUa3dlak5pTUV0dll6QkRVMlpsWHpoVFdUbHFSR0o1YkRkM1kydEhMVTVwWDFCdFdXOUhjR0Z6ZEMxbldVcEhUMjkzUm1saGRXSkViVmR6ZFhwd1QxTnpOME54TW5KUloxQlBkME5QU1VWVWMybHlNbFZvUVV0NlVIZFVhMVV5UzNWUmJHRldkRmszU1dKd1pVUllVMkZFVG1aV1ZsRnVUMGhsZFd0T01sVndTbGd3TkhweVdDMVBTRUphV25GNk5Yb3dYMWRCVTFnMlltODBPRmhIV205WlQwNW1YMjV1UlVKTWNucHNNSGR5Y1hKaFltUkVkblJYZG1Kc1FVaHFUV3BwTkc5R1pUQkVlbGw2ZHpSM2FISlBTSFJoYjJGbVMwNTBiV1pxV2pCSVNWWnZTalpRT0RoclVGVmhia1p5VFhsaWFTMVBjREZZV1dSTFdERkZjSHB0ZUhseWFtRXdNR1JmTkhOWmFEVlZTbVZ1ZUVkRU1XRlFhbU4xVERabk4wdDVSSGxHU2xsT1VEQlJXR1ZLTUhGM1UwWkJTSE5oWkRWQ2NXZHNaMFpqYW1ST1YxZFlhMDVOVUZSSFZWVktRekZSYVhodlUxTm1SV1EwTUdsdWNEWXlPV1YwUjNkcGFVcEVTM040YUZadmRXbHJhblkyZFdFelNHWXpUV3hMYURCa2JIRTFSblJ4Wms4NU1XbGtOM0pHYjBGeU4xZFJNMU5qYkZCd05rZE9jV1JqT1hGRGIyNU5Xak5TUlhkemFsUXRObGt4UWxkUE16ZGFaRTlxVGtaZlIweEhRbXRNWXpCWE9GUjNOMHBsYVhwS2RtSlZkMmxGTVhCbVNIWkdkVTFJY0MxbFdYSkVZM0V0ZFROWWRtVlFlV3hhYlVKMmVreGZUMGxOU2xaSlRFTlBZMVpEUjFwd1RHZFhZMmhIYVVKakxUSmFabXd0U1RNeFJEWkhlSGhYTkhOMU1GZGhOMjFCVlVnNGNFTlJXSGx2WW5ScWNUaHZXWGxKT1d0TVRXc3lRMWc0Um5wU2JEVjBlRGxpTW5vMVRYaEtkelExY201S1JHSmZkamhmTlhOWmRGYzRjak5FVVdkMlpXVnNRWEJyZW5OdFpHcEljVGhWYzFsZkxWa3dRVTkyTVZVMmIyMTNVeTFLVEUxeFIwUldRbmc0VEdsTlpGVktjVmxzTkZGa1UwazFabE0wZUhsRk5WZ3lWR0ZaYzJadlYyaHRPRFpzTjNCT1dHRnBiMHhUVDBkMmRuZFVOMlptVm05dWIwRTFZVkZuYldKNmIwMUNaMng2VGkxSk56bHhXV3BJVGt4RFYwVllUM05pTVcwemRHc3lUVWN6TVVKcVRHdElNVWg1YmtKQmVrbFNVMnczZEVKUlJGOUlNVWRyZERsbFJraHVYekJXZUhGbE1rTTBlVE40YVU1T1pFcGpVMkpFZFMxWVdITjNTMnhWVjJwYVgzVXRXbGcwZG5OSE1qUXpYMlJHTVhSV1kxWkZRMlZwU25OdVlXTkdVek5wVUd4b2FUbDVSRVp4YVhsbFRqbG1aRWxYVFZCMVFWbG9OMEl3TW5KV1JUVjRkREJLZG5obmJGZHhSVlY1ZWpjMFIyeGlZemRIVmkxeFpESmlaMnhFZGxkcVRuSjZNVEZWUkRWamVIQlFkRk5DVmtSU2RITlRaSGhWZG05WE9VUkNhWEYwTm1kSFRtb3RNV1pNYlhSeVJWTnJhRWhIVDB0SU0yVkxUbFZ2V1VGNlJTMDJialJZYkRKdFFUVnJhRVJ4WmpjeFptcERNR001UmpkM2QwNW1VRXd5YUZCZlEwWjFSbEUzY0doRk5ISkZZMWxTTWs5d2RXRnhiRzFrYjBVMmIxWkJaRzkyU2xneFZWOXNiMDVWWkUxRFJ6QjBjWGhpVjBVMldYY3pTUzF4UVcxa1RuZEJRVGRvWVZFNGNsSTBaVUl0UmxacVdETnJXazVLY21aRk9HVndRbWxqUjB0blRFZEZVR3N6YzJOclkwSTNlVlZZVEdkcE1YQkdiMHAyZVU1aGRVZFdVblJQYVhaQlZtdHZSa0UzTFU1Sk1XaFJRMUpMV2kxSWJ6WkxjWEkxZGtSTWJsOVdUa0ZFVmpKZmMwUlFWV3gwUTJ0TFRsbDJaM2gxZFVOSVkzbEVORUpRZVUxMVREQnpOMVowWDI1MWRrVmlUMU54TkRkUk5rVjViMEpRTUZGNmR6RlJSR2RxY1U1eVgwNTBjMDkxWm14R2NUVjBlRkJGT1dGVmFXeFJTMEZYYldwQlVVbHNOVmgwZERZdGFFRlViMWxmUjFWc1EycG1WVkJQV0hkcFVRPT0aIBIaVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1FIZGgDKGM%3D")).to eq("4qmFsgKXFBIYVUMtbEhKWlIzR3F4bTI0X1ZkX0FKNVl3GvoTRWdsamIyMXRkVzVwZEhtNEFRQ3FBLWNPQ3NBT1VWVlNWRk5zT1hCTldFWXhZbFZhYmxGWGFHRmlXRkpOVFc1V00xWkhTWGRQVlU1RVdUTm9lRkpXV2xWalJXUkdWVEJPYTFwclRrdGpWVW95V2pCWmVtTkVaSFJQVjJjd1YxaFdiVkp0YUZWUFdGSndWakphVVU0eFRYbE5SV1JhWWxad1NWRlVhM2RsYWs1cFRVVjBkbGw2UWtSVk1scHNXSHBvVkZkVWJIRlNSMG8xWWtSa00xa3lkRWhNVlRWd1dERkNkRmRYT1VoalIwWjZaRU14YmxkVmNFaFVNamt6VW0xc2FHUlhTa1ZpVm1SNlpGaHdkMVF4VG5wT01FNTRUVzVLVWxveFFsQmtNRTVRVTFWV1ZXTXliSGxOYkZadlVWVjBObFZJWkZWaE1WVjVVek5XVW1KSFJsZGtSbXN6VTFkS2QxcFZVbGxWTWtaRlZHMWFWMVpzUm5WVU1HaHNaRmQwVDAxc1ZuZFRiR2QzVGtod2VWZERNVkJUUlVwaFYyNUdOazVZYjNkWU1XUkNWVEZuTWxsdE9EQlBSbWhJVjIwNVdsUXdOVzFZTWpWMVVsVktUV051Y0hOTlNHUjVZMWhLYUZsdFVrVmtibEpZWkcxS2MxRlZhSEZVVjNCd1RrYzVSMXBVUWtWbGJHdzJaSHBTTTJGSVNsQlRTRkpvWWpKR2JWTXdOVEJpVjFweFYycENTVk5XV25aVGFscFJUMFJvY2xWR1ZtaGlhMXA1VkZoc2FXRlRNVkJqUkVaWlYxZFNURmRFUmtaalNIQjBaVWhzZVdGdFJYZE5SMUptVGtoT1dtRkVWbFpUYlZaMVpVVmtSVTFYUmxGaGJVNHhWRVJhYms0d2REVlNTR3hIVTJ4c1QxVkVRbEpYUjFaTFRVaEdNMVV3V2tKVFNFNW9Xa1JXUTJOWFpITmFNRnBxWVcxU1QxWXhaRmxoTURWT1ZVWlNTRlpXVmt0UmVrWlNZVmhvZGxVeFRtMVNWMUV3VFVkc2RXTkVXWGxQVjFZd1VqTmtjR0ZWY0VWVE0wNDBZVVphZG1SWGJISmhibGt5WkZkRmVsTkhXWHBVVjNoTVlVUkNhMkpJUlRGU2JsSjRXbXM0TlUxWGJHdE9NMHBIWWpCR2VVNHhaRkpOTVU1cVlrWkNkMDVyWkU5alYxSnFUMWhHUkdJeU5VNVhhazVUVWxoa2VtRnNVWFJPYkd0NFVXeGtVRTE2WkdGYVJUbHhWR3RhWmxJd2VFaFJiWFJOV1hwQ1dFOUdVak5PTUhCc1lWaHdTMlJ0U2xaa01teEdUVmhDYlZOSVdrZGtWVEZKWTBNeGJGZFlTa1ZaTTBWMFpGUk9XV1J0VmxGbFYzaGhZbFZLTW1WcmVHWlVNR3hPVTJ4YVNsUkZUbEJaTVZwRVVqRndkMVJIWkZoWk1taElZVlZLYWt4VVNtRmFiWGQwVTFSTmVGSkVXa2hsU0doWVRraE9NVTFHWkdoT01qRkNWbFZuTkdORlRsSlhTR3gyV1c1U2NXTlVhSFpYV0d4S1QxZDBUVlJYYzNsUk1XYzBVbTV3VTJKRVZqQmxSR3hwVFc1dk1WUllhRXRrZWxFeFkyMDFTMUpIU21aa2FtaG1UbGhPV21SR1l6UmphazVGVlZka01scFhWbk5SV0VKeVpXNU9kRnBIY0VsalZHaFdZekZzWmt4V2EzZFJWVGt5VFZaVk1tSXlNVE5WZVRGTFZFVXhlRkl3VWxkUmJtYzBWRWRzVGxwR1ZrdGpWbXh6VGtaR2ExVXdhekZhYkUwd1pVaHNSazVXWjNsV1IwWmFZekphZGxZeWFIUlBSRnB6VGpOQ1QxZEhSbkJpTUhoVVZEQmtNbVJ1WkZWT01scHRWbTA1ZFdJd1JURlpWa1p1WWxkS05tSXdNVU5hTW5nMlZHa3hTazU2YkhoWFYzQkpWR3Q0UkZZd1ZsbFVNMDVwVFZjd2VtUkhjM2xVVldONlRWVktjVlJIZEVsTlZXZzFZbXRLUW1WcmJGTlZNbmN6WkVWS1VsSkdPVWxOVldSeVpFUnNiRkpyYUhWWWVrSlhaVWhHYkUxclRUQmxWRTQwWVZVMVQxcEZjR3BWTWtwRlpGTXhXVmRJVGpOVE1uaFdWakp3WVZnelZYUlhiR2N3Wkc1T1NFMXFVWHBZTWxKSFRWaFNWMWt4V2taUk1sWndVMjVPZFZsWFRrZFZlazV3VlVkNGIyRlViRFZTUlZwNFlWaHNiRlJxYkcxYVJXeFlWRlpDTVZGV2JHOU9NRWwzVFc1S1YxSlVWalJrUkVKTFpHNW9ibUpHWkhoU1ZsWTFaV3BqTUZJeWVHbFplbVJJVm1reGVGcEVTbWxhTW5oRlpHeGtjVlJ1U2paTlZFWldVa1JXYW1WSVFsRmtSazVEVm10U1UyUklUbFJhU0doV1pHMDVXRTlWVWtOaFdFWXdUbTFrU0ZSdGIzUk5WMXBOWWxoU2VWSldUbkpoUldoSVZEQjBTVTB5Vmt4VWJGWjJWMVZHTmxKVE1ESmlhbEpaWWtSS2RGRlVWbkpoUlZKNFdtcGplRnB0Y0VSTlIwMDFVbXBrTTJRd05XMVZSWGQ1WVVaQ1psRXdXakZTYkVVelkwZG9SazVJU2taWk1XeFRUV3M1ZDJSWFJuaGlSekZyWWpCVk1tSXhXa0phUnpreVUyeG5lRlpXT1hOaU1EVldXa1V4UkZKNlFqQmpXR2hwVmpCVk1sZFlZM3BUVXpGNFVWY3hhMVJ1WkVKUlZHUnZXVlpGTkdOc1NUQmFWVWwwVW14YWNWZEVUbkpYYXpWTFkyMWFSazlIVm5kUmJXeHFVakIwYmxSRlpFWlZSM042WXpKT2Nsa3dTVE5sVmxaWlZFZGtjRTFZUWtkaU1IQXlaVlUxYUdSVlpGZFZibEpRWVZoYVFsWnRkSFpTYTBVelRGVTFTazFYYUZKUk1VcE1WMmt4U1dKNldreGpXRWt4Wkd0U1RXSnNPVmRVYTBaRlZtcEtabU13VWxGV1YzZ3dVVEowVEZSc2JESmFNMmd4WkZWT1NWa3piRVZPUlVwUlpWVXhNVlJFUW5wT01Wb3dXREkxTVdSclZtbFVNVTU0VGtSa1VrNXJWalZpTUVwUlRVWkdObVI2UmxKU1IyUnhZMVUxZVZnd05UQmpNRGt4V20xNFIyTlVWakJsUmtKR1QxZEdWbUZYZUZKVE1FWllZbGR3UWxWVmJITk9WbWd3WkVSWmRHRkZSbFZpTVd4bVVqRldjMUV5Y0cxV1ZrSlFWMGhrY0ZWUlBUMGFJQklhVldkNVJUSTJOVzFyVWtrMmNFOXVTMjFuYkRSQllVRkNRMUZJWkdnREtHTSUzRA%3D%3D") - end - end - - describe "#extract_channel_community_cursor" do - it "correctly extracts a community cursor from a given continuation" do - expect(extract_channel_community_cursor("4qmFsgIsEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaEEVnbGpiMjF0ZFc1cGRIbTQ%3D")).to eq("Egljb21tdW5pdHk=") - expect(extract_channel_community_cursor("4qmFsgJoEhhVQ0NqOTU2SUY2MkZiVDdHb3VzemFqOXcaTEVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2QzY0U5TlFtVndXRWRqY2xoc1VIZzJXalJCWVVGQ1ExRklaR2dES0FBJTI1M0Q%3D")).to eq("Egljb21tdW5pdHm4AQCqAyQaIEhkaAMSGlVnd3BPTUJlcFhHY3JYbFB4Nlo0QWFBQkNRKAA=") - - expect(extract_channel_community_cursor("4qmFsgJoEhhVQy1sSEpaUjNHcXhtMjRfVmRfQUo1WXcaTEVnbGpiMjF0ZFc1cGRIbTRBUUNxQXlRYUlCSWFWV2Q1UlRJMk5XMXJVa2syY0U5dVMyMW5iRFJCWVVGQ1ExRklaR2dES0FBJTI1M0Q%3D")).to eq("Egljb21tdW5pdHm4AQCqAyQaIEhkaAMSGlVneUUyNjVta1JJNnBPbkttZ2w0QWFBQkNRKAA=") - expect(extract_channel_community_cursor("4qmFsgKZFBIYVUMtbEhKWlIzR3F4bTI0X1ZkX0FKNVl3GvwTRWdsamIyMXRkVzVwZEhtNEFRQ3FBLWNPQ3NBT1VWVlNWRk5zT1hCTldFWXhZbFZhYmxGWGFHRmlXRkpOVFc1V00xWkhTWGRQVlU1RVdUTm9lRkpXV2xWalJXUkdWVEJPYTFwclRrdGpWVW95V2pCWmVtTkVaSFJQVjJjd1YxaFdiVkp0YUZWUFdGSndWakphVVU0eFRYbE5SV1JhWWxad1NWRlVhM2RsYWs1cFRVVjBkbGw2UWtSVk1scHNXSHBvVkZkVWJIRlNSMG8xWWtSa00xa3lkRWhNVlRWd1dERkNkRmRYT1VoalIwWjZaRU14YmxkVmNFaFVNamt6VW0xc2FHUlhTa1ZpVm1SNlpGaHdkMVF4VG5wT01FNTRUVzVLVWxveFFsQmtNRTVRVTFWV1ZXTXliSGxOYkZadlVWVjBObFZJWkZWaE1WVjVVek5XVW1KSFJsZGtSbXN6VTFkS2QxcFZVbGxWTWtaRlZHMWFWMVpzUm5WVU1HaHNaRmQwVDAxc1ZuZFRiR2QzVGtod2VWZERNVkJUUlVwaFYyNUdOazVZYjNkWU1XUkNWVEZuTWxsdE9EQlBSbWhJVjIwNVdsUXdOVzFZTWpWMVVsVktUV051Y0hOTlNHUjVZMWhLYUZsdFVrVmtibEpZWkcxS2MxRlZhSEZVVjNCd1RrYzVSMXBVUWtWbGJHdzJaSHBTTTJGSVNsQlRTRkpvWWpKR2JWTXdOVEJpVjFweFYycENTVk5XV25aVGFscFJUMFJvY2xWR1ZtaGlhMXA1VkZoc2FXRlRNVkJqUkVaWlYxZFNURmRFUmtaalNIQjBaVWhzZVdGdFJYZE5SMUptVGtoT1dtRkVWbFpUYlZaMVpVVmtSVTFYUmxGaGJVNHhWRVJhYms0d2REVlNTR3hIVTJ4c1QxVkVRbEpYUjFaTFRVaEdNMVV3V2tKVFNFNW9Xa1JXUTJOWFpITmFNRnBxWVcxU1QxWXhaRmxoTURWT1ZVWlNTRlpXVmt0UmVrWlNZVmhvZGxVeFRtMVNWMUV3VFVkc2RXTkVXWGxQVjFZd1VqTmtjR0ZWY0VWVE0wNDBZVVphZG1SWGJISmhibGt5WkZkRmVsTkhXWHBVVjNoTVlVUkNhMkpJUlRGU2JsSjRXbXM0TlUxWGJHdE9NMHBIWWpCR2VVNHhaRkpOTVU1cVlrWkNkMDVyWkU5alYxSnFUMWhHUkdJeU5VNVhhazVUVWxoa2VtRnNVWFJPYkd0NFVXeGtVRTE2WkdGYVJUbHhWR3RhWmxJd2VFaFJiWFJOV1hwQ1dFOUdVak5PTUhCc1lWaHdTMlJ0U2xaa01teEdUVmhDYlZOSVdrZGtWVEZKWTBNeGJGZFlTa1ZaTTBWMFpGUk9XV1J0VmxGbFYzaGhZbFZLTW1WcmVHWlVNR3hPVTJ4YVNsUkZUbEJaTVZwRVVqRndkMVJIWkZoWk1taElZVlZLYWt4VVNtRmFiWGQwVTFSTmVGSkVXa2hsU0doWVRraE9NVTFHWkdoT01qRkNWbFZuTkdORlRsSlhTR3gyV1c1U2NXTlVhSFpYV0d4S1QxZDBUVlJYYzNsUk1XYzBVbTV3VTJKRVZqQmxSR3hwVFc1dk1WUllhRXRrZWxFeFkyMDFTMUpIU21aa2FtaG1UbGhPV21SR1l6UmphazVGVlZka01scFhWbk5SV0VKeVpXNU9kRnBIY0VsalZHaFdZekZzWmt4V2EzZFJWVGt5VFZaVk1tSXlNVE5WZVRGTFZFVXhlRkl3VWxkUmJtYzBWRWRzVGxwR1ZrdGpWbXh6VGtaR2ExVXdhekZhYkUwd1pVaHNSazVXWjNsV1IwWmFZekphZGxZeWFIUlBSRnB6VGpOQ1QxZEhSbkJpTUhoVVZEQmtNbVJ1WkZWT01scHRWbTA1ZFdJd1JURlpWa1p1WWxkS05tSXdNVU5hTW5nMlZHa3hTazU2YkhoWFYzQkpWR3Q0UkZZd1ZsbFVNMDVwVFZjd2VtUkhjM2xVVldONlRWVktjVlJIZEVsTlZXZzFZbXRLUW1WcmJGTlZNbmN6WkVWS1VsSkdPVWxOVldSeVpFUnNiRkpyYUhWWWVrSlhaVWhHYkUxclRUQmxWRTQwWVZVMVQxcEZjR3BWTWtwRlpGTXhXVmRJVGpOVE1uaFdWakp3WVZnelZYUlhiR2N3Wkc1T1NFMXFVWHBZTWxKSFRWaFNWMWt4V2taUk1sWndVMjVPZFZsWFRrZFZlazV3VlVkNGIyRlViRFZTUlZwNFlWaHNiRlJxYkcxYVJXeFlWRlpDTVZGV2JHOU9NRWwzVFc1S1YxSlVWalJrUkVKTFpHNW9ibUpHWkhoU1ZsWTFaV3BqTUZJeWVHbFplbVJJVm1reGVGcEVTbWxhTW5oRlpHeGtjVlJ1U2paTlZFWldVa1JXYW1WSVFsRmtSazVEVm10U1UyUklUbFJhU0doV1pHMDVXRTlWVWtOaFdFWXdUbTFrU0ZSdGIzUk5WMXBOWWxoU2VWSldUbkpoUldoSVZEQjBTVTB5Vmt4VWJGWjJWMVZHTmxKVE1ESmlhbEpaWWtSS2RGRlVWbkpoUlZKNFdtcGplRnB0Y0VSTlIwMDFVbXBrTTJRd05XMVZSWGQ1WVVaQ1psRXdXakZTYkVVelkwZG9SazVJU2taWk1XeFRUV3M1ZDJSWFJuaGlSekZyWWpCVk1tSXhXa0phUnpreVUyeG5lRlpXT1hOaU1EVldXa1V4UkZKNlFqQmpXR2hwVmpCVk1sZFlZM3BUVXpGNFVWY3hhMVJ1WkVKUlZHUnZXVlpGTkdOc1NUQmFWVWwwVW14YWNWZEVUbkpYYXpWTFkyMWFSazlIVm5kUmJXeHFVakIwYmxSRlpFWlZSM042WXpKT2Nsa3dTVE5sVmxaWlZFZGtjRTFZUWtkaU1IQXlaVlUxYUdSVlpGZFZibEpRWVZoYVFsWnRkSFpTYTBVelRGVTFTazFYYUZKUk1VcE1WMmt4U1dKNldreGpXRWt4Wkd0U1RXSnNPVmRVYTBaRlZtcEtabU13VWxGV1YzZ3dVVEowVEZSc2JESmFNMmd4WkZWT1NWa3piRVZPUlVwUlpWVXhNVlJFUW5wT01Wb3dXREkxTVdSclZtbFVNVTU0VGtSa1VrNXJWalZpTUVwUlRVWkdObVI2UmxKU1IyUnhZMVUxZVZnd05UQmpNRGt4V20xNFIyTlVWakJsUmtKR1QxZEdWbUZYZUZKVE1FWllZbGR3UWxWVmJITk9WbWd3WkVSWmRHRkZSbFZpTVd4bVVqRldjMUV5Y0cxV1ZrSlFWMGhrY0ZWUlBUMGFJQklhVldkNVJUSTJOVzFyVWtrMmNFOXVTMjFuYkRSQllVRkNRMUZJWkdnREtHTSUyNTNE")).to eq("Egljb21tdW5pdHm4AQCqA-kOCsAOUVVSVFNsOXBNWEYxYlVablFXaGFiWFJNTW5WM1ZHSXdPVU5EWTNoeFJWWlVjRWRGVTBOa1prTktjVUoyWjBZemNEZHRPV2cwV1hWbVJtaFVPWFJwVjJaUU4xTXlNRWRaYlZwSVFUa3dlak5pTUV0dll6QkRVMlpsWHpoVFdUbHFSR0o1YkRkM1kydEhMVTVwWDFCdFdXOUhjR0Z6ZEMxbldVcEhUMjkzUm1saGRXSkViVmR6ZFhwd1QxTnpOME54TW5KUloxQlBkME5QU1VWVWMybHlNbFZvUVV0NlVIZFVhMVV5UzNWUmJHRldkRmszU1dKd1pVUllVMkZFVG1aV1ZsRnVUMGhsZFd0T01sVndTbGd3TkhweVdDMVBTRUphV25GNk5Yb3dYMWRCVTFnMlltODBPRmhIV205WlQwNW1YMjV1UlVKTWNucHNNSGR5Y1hKaFltUkVkblJYZG1Kc1FVaHFUV3BwTkc5R1pUQkVlbGw2ZHpSM2FISlBTSFJoYjJGbVMwNTBiV1pxV2pCSVNWWnZTalpRT0RoclVGVmhia1p5VFhsaWFTMVBjREZZV1dSTFdERkZjSHB0ZUhseWFtRXdNR1JmTkhOWmFEVlZTbVZ1ZUVkRU1XRlFhbU4xVERabk4wdDVSSGxHU2xsT1VEQlJXR1ZLTUhGM1UwWkJTSE5oWkRWQ2NXZHNaMFpqYW1ST1YxZFlhMDVOVUZSSFZWVktRekZSYVhodlUxTm1SV1EwTUdsdWNEWXlPV1YwUjNkcGFVcEVTM040YUZadmRXbHJhblkyZFdFelNHWXpUV3hMYURCa2JIRTFSblJ4Wms4NU1XbGtOM0pHYjBGeU4xZFJNMU5qYkZCd05rZE9jV1JqT1hGRGIyNU5Xak5TUlhkemFsUXRObGt4UWxkUE16ZGFaRTlxVGtaZlIweEhRbXRNWXpCWE9GUjNOMHBsYVhwS2RtSlZkMmxGTVhCbVNIWkdkVTFJY0MxbFdYSkVZM0V0ZFROWWRtVlFlV3hhYlVKMmVreGZUMGxOU2xaSlRFTlBZMVpEUjFwd1RHZFhZMmhIYVVKakxUSmFabXd0U1RNeFJEWkhlSGhYTkhOMU1GZGhOMjFCVlVnNGNFTlJXSGx2WW5ScWNUaHZXWGxKT1d0TVRXc3lRMWc0Um5wU2JEVjBlRGxpTW5vMVRYaEtkelExY201S1JHSmZkamhmTlhOWmRGYzRjak5FVVdkMlpXVnNRWEJyZW5OdFpHcEljVGhWYzFsZkxWa3dRVTkyTVZVMmIyMTNVeTFLVEUxeFIwUldRbmc0VEdsTlpGVktjVmxzTkZGa1UwazFabE0wZUhsRk5WZ3lWR0ZaYzJadlYyaHRPRFpzTjNCT1dHRnBiMHhUVDBkMmRuZFVOMlptVm05dWIwRTFZVkZuYldKNmIwMUNaMng2VGkxSk56bHhXV3BJVGt4RFYwVllUM05pTVcwemRHc3lUVWN6TVVKcVRHdElNVWg1YmtKQmVrbFNVMnczZEVKUlJGOUlNVWRyZERsbFJraHVYekJXZUhGbE1rTTBlVE40YVU1T1pFcGpVMkpFZFMxWVdITjNTMnhWVjJwYVgzVXRXbGcwZG5OSE1qUXpYMlJHTVhSV1kxWkZRMlZwU25OdVlXTkdVek5wVUd4b2FUbDVSRVp4YVhsbFRqbG1aRWxYVFZCMVFWbG9OMEl3TW5KV1JUVjRkREJLZG5obmJGZHhSVlY1ZWpjMFIyeGlZemRIVmkxeFpESmlaMnhFZGxkcVRuSjZNVEZWUkRWamVIQlFkRk5DVmtSU2RITlRaSGhWZG05WE9VUkNhWEYwTm1kSFRtb3RNV1pNYlhSeVJWTnJhRWhIVDB0SU0yVkxUbFZ2V1VGNlJTMDJialJZYkRKdFFUVnJhRVJ4WmpjeFptcERNR001UmpkM2QwNW1VRXd5YUZCZlEwWjFSbEUzY0doRk5ISkZZMWxTTWs5d2RXRnhiRzFrYjBVMmIxWkJaRzkyU2xneFZWOXNiMDVWWkUxRFJ6QjBjWGhpVjBVMldYY3pTUzF4UVcxa1RuZEJRVGRvWVZFNGNsSTBaVUl0UmxacVdETnJXazVLY21aRk9HVndRbWxqUjB0blRFZEZVR3N6YzJOclkwSTNlVlZZVEdkcE1YQkdiMHAyZVU1aGRVZFdVblJQYVhaQlZtdHZSa0UzTFU1Sk1XaFJRMUpMV2kxSWJ6WkxjWEkxZGtSTWJsOVdUa0ZFVmpKZmMwUlFWV3gwUTJ0TFRsbDJaM2gxZFVOSVkzbEVORUpRZVUxMVREQnpOMVowWDI1MWRrVmlUMU54TkRkUk5rVjViMEpRTUZGNmR6RlJSR2RxY1U1eVgwNTBjMDkxWm14R2NUVjBlRkJGT1dGVmFXeFJTMEZYYldwQlVVbHNOVmgwZERZdGFFRlViMWxmUjFWc1EycG1WVkJQV0hkcFVRPT0aIhIcVWd5RTI2NW1rUkk2cE9uS21nbDRBYUFCQ1E9PUhkaAMoYw==") - end - end - - describe "#sign_token" do - it "correctly signs a given hash" do - token = { - "session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - "expires" => 1554680038, - "scopes" => [ - ":notifications", - ":subscriptions/*", - "GET:tokens*", - ], - "signature" => "f__2hS20th8pALF305PJFK-D2aVtvefNnQheILHD2vU=", - } - expect(sign_token("SECRET_KEY", token)).to eq(token["signature"]) - - token = { - "session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - "scopes" => [":notifications", "POST:subscriptions/*"], - "signature" => "fNvXoT0MRAL9eE6lTE33CEg8HitYJDOL9a22rSN2Ihg=", - } - expect(sign_token("SECRET_KEY", token)).to eq(token["signature"]) - end - end -end diff --git a/spec/invidious/search/ctoken_spec.cr b/spec/invidious/search/ctoken_spec.cr new file mode 100644 index 000000000..97ec9a22e --- /dev/null +++ b/spec/invidious/search/ctoken_spec.cr @@ -0,0 +1,11 @@ +require "../../spec_helper" + +Spectator.describe Invidious::Search::CToken do + describe ".produce_channel_search_continuation" do + it "correctly produces token for searching a specific channel" do + expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") + + expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "По ожиशुपतिरपि子而時ஸ்றீனி", 0)).to eq("4qmFsgKoARIYVUNYdXFTQmxIQUU2WHcteWVKQTBUdW53GiBFZ1p6WldGeVkyZ3dBVGdCWUFGNkJFZEJRVDI0QVFBPVo-0J_QviDQvtC20LjgpLbgpYHgpKrgpKTgpL_gpLDgpKrgpL_lrZDogIzmmYLgrrjgr43grrHgr4Dgrqngrr-aAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D") + end + end +end diff --git a/spec/invidious/hashtag_spec.cr b/spec/invidious/search/hashtag_spec.cr similarity index 99% rename from spec/invidious/hashtag_spec.cr rename to spec/invidious/search/hashtag_spec.cr index abc81225c..813646c60 100644 --- a/spec/invidious/hashtag_spec.cr +++ b/spec/invidious/search/hashtag_spec.cr @@ -1,4 +1,4 @@ -require "../parsers_helper.cr" +require "../../parsers_helper.cr" Spectator.describe Invidious::Hashtag do it "parses richItemRenderer containers (test 1)" do diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index b3060acfa..3635c1ea0 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -9,7 +9,6 @@ require "../src/invidious/videos/caption" require "../src/invidious/videos" require "../src/invidious/playlists" require "../src/invidious/search/ctoken" -require "../src/invidious/trending" require "spectator" Spectator.configure do |config| diff --git a/src/invidious.cr b/src/invidious.cr index cc65dde0e..9811b8d7b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -44,6 +44,8 @@ require "./invidious/jsonify/**" require "./invidious/*" require "./invidious/comments/*" require "./invidious/channels/*" +require "./invidious/playlists/*" +require "./invidious/feeds/*" require "./invidious/user/*" require "./invidious/search/*" require "./invidious/routes/**" @@ -230,7 +232,7 @@ error 404 do |env| end error 500 do |env, exception| - error_template(500, exception) + Errors.error_template(500, exception) end # Init Kemal diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 247f628a0..759d89380 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -19,202 +19,206 @@ record AboutChannel, verified : Bool, is_age_gated : Bool -def get_about_info(ucid) : AboutChannel - begin - # Fetch channel information from channel home page - initdata = YoutubeAPI.browse(browse_id: ucid, params: "") - rescue - raise InfoException.new("Could not get channel info.") - end +module Invidious::Channels::About + extend self - if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" - error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s - if error_message == "This channel does not exist." - raise NotFoundException.new(error_message) - else - raise InfoException.new(error_message) + def get_about_info(ucid) : AboutChannel + begin + # Fetch channel information from channel home page + initdata = YoutubeAPI.browse(browse_id: ucid, params: "") + rescue + raise InfoException.new("Could not get channel info.") end - end - if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? - raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) - end + if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" + error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s + if error_message == "This channel does not exist." + raise NotFoundException.new(error_message) + else + raise InfoException.new(error_message) + end + end - auto_generated = false - # Check for special auto generated gaming channels - if !initdata.has_key?("metadata") - auto_generated = true - end + if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? + raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) + end - tags = [] of String - tab_names = [] of String - total_views = 0_i64 - joined = Time.unix(0) - - if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") - description_node = nil - author = age_gate_renderer["channelTitle"].as_s - ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s - author_url = "https://www.youtube.com/channel/#{ucid}" - author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s - banner = nil - is_family_friendly = false - is_age_gated = true - tab_names = ["videos", "shorts", "streams"] auto_generated = false - else - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + # Check for special auto generated gaming channels + if !initdata.has_key?("metadata") + auto_generated = true + end - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node - - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = age_gate_renderer["channelTitle"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName") - .try &.as_s - # CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For - # music/artist channels - # TODO: Maybe separate verified author from verified artist? - author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - # Raises a KeyError on failure. - # TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") - banner = banners.try &.[-1]?.try &.["url"].as_s? + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String - end + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName") + .try &.as_s + # CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For + # music/artist channels + # TODO: Maybe separate verified author from verified artist? + author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + # Raises a KeyError on failure. + # TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end + + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String end - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || - channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end end end - end - allowed_regions = initdata - .dig?("microformat", "microformatDataRenderer", "availableCountries") - .try &.as_a.map(&.as_s) || [] of String + allowed_regions = initdata + .dig?("microformat", "microformatDataRenderer", "availableCountries") + .try &.as_a.map(&.as_s) || [] of String - description = !description_node.nil? ? description_node.as_s : "" - description_html = HTML.escape(description) + description = !description_node.nil? ? description_node.as_s : "" + description_html = HTML.escape(description) - if !description_node.nil? - if description_node.as_h?.nil? - description_node = text_to_parsed_content(description_node.as_s) - end - description_html = parse_content(description_node) - if description_html == "" && description != "" - description_html = HTML.escape(description) - end - end - - sub_count = 0 - pronouns = nil - - if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) - metadata_rows.each do |row| - subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } - if !subscribe_metadata_part.nil? - sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + if !description_node.nil? + if description_node.as_h?.nil? + description_node = text_to_parsed_content(description_node.as_s) end - - pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") } - if !pronoun_metadata_part.nil? - pronouns = pronoun_metadata_part.dig("text", "content").as_s + description_html = parse_content(description_node) + if description_html == "" && description != "" + description_html = HTML.escape(description) end - - break if sub_count != 0 && !pronouns.nil? end + + sub_count = 0 + pronouns = nil + + if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) + metadata_rows.each do |row| + subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !subscribe_metadata_part.nil? + sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + end + + pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") } + if !pronoun_metadata_part.nil? + pronouns = pronoun_metadata_part.dig("text", "content").as_s + end + + break if sub_count != 0 && !pronouns.nil? + end + end + + AboutChannel.new( + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description: description, + description_html: description_html, + total_views: total_views, + sub_count: sub_count, + joined: joined, + is_family_friendly: is_family_friendly, + pronouns: pronouns, + allowed_regions: allowed_regions, + tabs: tab_names, + tags: tags, + verified: author_verified || false, + is_age_gated: is_age_gated || false, + ) end - AboutChannel.new( - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description: description, - description_html: description_html, - total_views: total_views, - sub_count: sub_count, - joined: joined, - is_family_friendly: is_family_friendly, - pronouns: pronouns, - allowed_regions: allowed_regions, - tabs: tab_names, - tags: tags, - verified: author_verified || false, - is_age_gated: is_age_gated || false, - ) -end - -def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?} - if continuation.nil? - # params is {"2:string":"channels"} encoded - initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation) - end - - items, continuation = extract_items(initial_data) - - return items.select(SearchChannel), continuation + def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?} + if continuation.nil? + # params is {"2:string":"channels"} encoded + initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") + else + initial_data = YoutubeAPI.browse(continuation) + end + + items, continuation = extract_items(initial_data) + + return items.select(SearchChannel), continuation + end end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 64f0484df..fa257bdd7 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -123,7 +123,7 @@ def get_batch_channels(channels) active_threads += 1 spawn do begin - get_channel(ucid) + Invidious::Channels::Channels.get_channel(ucid) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -144,153 +144,157 @@ def get_batch_channels(channels) return final end -def get_channel(id) : InvidiousChannel - channel = Invidious::Database::Channels.select(id) +module Invidious::Channels::Channels + extend self - if channel.nil? || (Time.utc - channel.updated) > 2.days - channel = fetch_channel(id, pull_all_videos: false) - Invidious::Database::Channels.insert(channel, update_on_conflict: true) + def get_channel(id) : InvidiousChannel + channel = Invidious::Database::Channels.select(id) + + if channel.nil? || (Time.utc - channel.updated) > 2.days + channel = self.fetch_channel(id, pull_all_videos: false) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) + end + + return channel end - return channel -end + def fetch_channel(ucid, pull_all_videos : Bool) + LOGGER.debug("fetch_channel: #{ucid}") + LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") -def fetch_channel(ucid, pull_all_videos : Bool) - LOGGER.debug("fetch_channel: #{ucid}") - LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "media" => "http://search.yahoo.com/mrss/", + "default" => "http://www.w3.org/2005/Atom", + } - namespaces = { - "yt" => "http://www.youtube.com/xml/schemas/2015", - "media" => "http://search.yahoo.com/mrss/", - "default" => "http://www.w3.org/2005/Atom", - } + LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") + rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body + LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") + rss = XML.parse(rss) - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse(rss) + author = rss.xpath_node("//default:feed/default:title", namespaces) + if !author + raise InfoException.new("Deleted or invalid channel") + end - author = rss.xpath_node("//default:feed/default:title", namespaces) - if !author - raise InfoException.new("Deleted or invalid channel") - end + author = author.content - author = author.content + # Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + if author.ends_with?(" - Topic") || + {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author + auto_generated = true + end - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end + LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") - - channel = InvidiousChannel.new({ - id: ucid, - author: author, - updated: Time.utc, - deleted: false, - subscribed: nil, - }) - - LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - videos, continuation = IV::Channel::Tabs.get_videos(channel) - - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| - video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content - title = entry.xpath_node("default:title", namespaces).not_nil!.content - - published = Time.parse_rfc3339( - entry.xpath_node("default:published", namespaces).not_nil!.content - ) - updated = Time.parse_rfc3339( - entry.xpath_node("default:updated", namespaces).not_nil!.content - ) - - author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content - ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content - - views = entry - .xpath_node("media:group/media:community/media:statistics", namespaces) - .try &.["views"]?.try &.to_i64? || 0_i64 - - channel_video = videos - .select(SearchVideo) - .select(&.id.== video_id)[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.badges.live_now? - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - - video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: updated, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, }) - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") + videos, continuation = IV::Channel::Tabs.get_videos(channel) - # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null - was_insert = Invidious::Database::ChannelVideos.insert(video) + LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + title = entry.xpath_node("default:title", namespaces).not_nil!.content - if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) - else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + published = Time.parse_rfc3339( + entry.xpath_node("default:published", namespaces).not_nil!.content + ) + updated = Time.parse_rfc3339( + entry.xpath_node("default:updated", namespaces).not_nil!.content + ) + + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content + + views = entry + .xpath_node("media:group/media:community/media:statistics", namespaces) + .try &.["views"]?.try &.to_i64? || 0_i64 + + channel_video = videos + .select(SearchVideo) + .select(&.id.== video_id)[0]? + + length_seconds = channel_video.try &.length_seconds + length_seconds ||= 0 + + live_now = channel_video.try &.badges.live_now? + live_now ||= false + + premiere_timestamp = channel_video.try &.premiere_timestamp + + video = ChannelVideo.new({ + id: video_id, + title: title, + published: published, + updated: updated, + ucid: ucid, + author: author, + length_seconds: length_seconds, + live_now: live_now, + premiere_timestamp: premiere_timestamp, + views: views, + }) + + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + + # We don't include the 'premiere_timestamp' here because channel pages don't include them, + # meaning the above timestamp is always null + was_insert = Invidious::Database::ChannelVideos.insert(video) + + if was_insert + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) + else + LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + end end - end - if pull_all_videos - loop do - # Keep fetching videos using the continuation token retrieved earlier - videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) + if pull_all_videos + loop do + # Keep fetching videos using the continuation token retrieved earlier + videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) - count = 0 - videos.select(SearchVideo).each do |video| - count += 1 - video = ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.badges.live_now?, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) + count = 0 + videos.select(SearchVideo).each do |video| + count += 1 + video = ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.badges.live_now?, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) - # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, - # so since they don't provide a published date here we can safely ignore them. - if Time.utc - video.published > 1.minute - was_insert = Invidious::Database::ChannelVideos.insert(video) - if was_insert - NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) + # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, + # so since they don't provide a published date here we can safely ignore them. + if Time.utc - video.published > 1.minute + was_insert = Invidious::Database::ChannelVideos.insert(video) + if was_insert + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) + end end end + + break if count < 25 + sleep 500.milliseconds end - - break if count < 25 - sleep 500.milliseconds end - end - channel.updated = Time.utc - return channel + channel.updated = Time.utc + return channel + end end diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 34a214265..947b8b5b5 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -1,198 +1,201 @@ -private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} +module Invidious::Channels::Community + extend self + private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} -# TODO: Add "sort_by" -def fetch_channel_community(ucid, cursor, locale, format, thin_mode) - if cursor.nil? - # EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts" - initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D") + # TODO: Add "sort_by" + def fetch_channel_community(ucid, cursor, locale, format, thin_mode) + if cursor.nil? + # EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts" + initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D") + + items = [] of JSON::Any + extract_items(initial_data) do |item| + items << item + end + else + continuation = self.produce_channel_community_continuation(ucid, cursor) + initial_data = YoutubeAPI.browse(continuation: continuation) + + container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents") + + raise InfoException.new("Can't extract community data") if container.nil? + + items = container.as_a + end + + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) + end + + def decode_ucid_from_post_protobuf(params) + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s) + end + + def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) + object = { + "56:embedded" => { + "2:string" => ucid, + "3:string" => post_id.to_s, + "11:string" => ucid, + }, + } + params = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + initial_data = YoutubeAPI.browse("FEpost_detail", params: params) items = [] of JSON::Any extract_items(initial_data) do |item| items << item end - else - continuation = produce_channel_community_continuation(ucid, cursor) - initial_data = YoutubeAPI.browse(continuation: continuation) - container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents") - - raise InfoException.new("Can't extract community data") if container.nil? - - items = container.as_a + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true) end - return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) -end - -def decode_ucid_from_post_protobuf(params) - decoded_protobuf = params.try { |i| URI.decode_www_form(i) } - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - - return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s) -end - -def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode) - object = { - "56:embedded" => { - "2:string" => ucid, - "3:string" => post_id.to_s, - "11:string" => ucid, - }, - } - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - initial_data = YoutubeAPI.browse("FEpost_detail", params: params) - - items = [] of JSON::Any - extract_items(initial_data) do |item| - items << item - end - - return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true) -end - -def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false) - if message = items[0]["messageRenderer"]? - error_message = (message["text"]["simpleText"]? || - message["text"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s || "" - if error_message == "This channel does not exist." - raise NotFoundException.new(error_message) - else - raise InfoException.new(error_message) - end - end - - response = JSON.build do |json| - json.object do - json.field "authorId", ucid - if is_single_post - json.field "singlePost", true + def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false) + if message = items[0]["messageRenderer"]? + error_message = (message["text"]["simpleText"]? || + message["text"]["runs"]?.try &.[0]?.try &.["text"]?) + .try &.as_s || "" + if error_message == "This channel does not exist." + raise NotFoundException.new(error_message) + else + raise InfoException.new(error_message) end - json.field "comments" do - json.array do - items.each do |post| - comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || - post["backstageCommentsContinuation"]? + end - post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || - post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? + response = JSON.build do |json| + json.object do + json.field "authorId", ucid + if is_single_post + json.field "singlePost", true + end + json.field "comments" do + json.array do + items.each do |post| + comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || + post["backstageCommentsContinuation"]? - next if !post + post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || + post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? - content_html = post["contentText"]?.try { |t| parse_content(t) } || "" - author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || "" + next if !post - json.object do - json.field "author", author - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s + content_html = post["contentText"]?.try { |t| parse_content(t) } || "" + author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || "" - qualities.each do |quality| - json.object do - json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") - json.field "width", quality - json.field "height", quality + json.object do + json.field "author", author + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s + + qualities.each do |quality| + json.object do + json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") + json.field "width", quality + json.field "height", quality + end end end end - end - if post["authorEndpoint"]? - json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] - json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - else - json.field "authorId", "" - json.field "authorUrl", "" - end + if post["authorEndpoint"]? + json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] + json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s + else + json.field "authorId", "" + json.field "authorUrl", "" + end - published_text = post["publishedTimeText"]["runs"][0]["text"].as_s - published = decode_date(published_text.rchop(" (edited)")) + published_text = post["publishedTimeText"]["runs"][0]["text"].as_s + published = decode_date(published_text.rchop(" (edited)")) - if published_text.includes?(" (edited)") - json.field "isEdited", true - else - json.field "isEdited", false - end + if published_text.includes?(" (edited)") + json.field "isEdited", true + else + json.field "isEdited", false + end - like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] - .try &.as_s.gsub(/\D/, "").to_i? || 0 + like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] + .try &.as_s.gsub(/\D/, "").to_i? || 0 - reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0") + reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0") - json.field "content", Helpers.html_to_content(content_html) - json.field "contentHtml", content_html + json.field "content", Invidious::Helpers.html_to_content(content_html) + json.field "contentHtml", content_html - json.field "published", published.to_unix - json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale)) + json.field "published", published.to_unix + json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale)) - json.field "likeCount", like_count - json.field "replyCount", reply_count - json.field "commentId", post["postId"]? || post["commentId"]? || "" - json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid + json.field "likeCount", like_count + json.field "replyCount", reply_count + json.field "commentId", post["postId"]? || post["commentId"]? || "" + json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid - if attachment = post["backstageAttachment"]? - json.field "attachment" do - case attachment.as_h - when .has_key?("videoRenderer") - parse_item(attachment) - .as(SearchVideo | ProblematicTimelineItem) - .to_json(locale, json) - when .has_key?("backstageImageRenderer") - json.object do - attachment = attachment["backstageImageRenderer"] - json.field "type", "image" + if attachment = post["backstageAttachment"]? + json.field "attachment" do + case attachment.as_h + when .has_key?("videoRenderer") + parse_item(attachment) + .as(SearchVideo | ProblematicTimelineItem) + .to_json(locale, json) + when .has_key?("backstageImageRenderer") + json.object do + attachment = attachment["backstageImageRenderer"] + json.field "type", "image" - json.field "imageThumbnails" do - json.array do - thumbnail = attachment["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") + json.field "imageThumbnails" do + json.array do + thumbnail = attachment["image"]["thumbnails"][0].as_h + width = thumbnail["width"].as_i + height = thumbnail["height"].as_i + aspect_ratio = (width.to_f / height.to_f) + url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - IMAGE_QUALITIES.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i + IMAGE_QUALITIES.each do |quality| + json.object do + json.field "url", url.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", (quality / aspect_ratio).ceil.to_i + end end end end end - end - when .has_key?("pollRenderer") - json.object do - attachment = attachment["pollRenderer"] - json.field "type", "poll" - json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) - json.field "choices" do - json.array do - attachment["choices"].as_a.each do |choice| - json.object do - json.field "text", choice.dig("text", "runs", 0, "text").as_s - # A choice can have an image associated with it. - # Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re - if choice["image"]? - thumbnail = choice["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - json.field "image" do - json.array do - IMAGE_QUALITIES.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i + when .has_key?("pollRenderer") + json.object do + attachment = attachment["pollRenderer"] + json.field "type", "poll" + json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) + json.field "choices" do + json.array do + attachment["choices"].as_a.each do |choice| + json.object do + json.field "text", choice.dig("text", "runs", 0, "text").as_s + # A choice can have an image associated with it. + # Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re + if choice["image"]? + thumbnail = choice["image"]["thumbnails"][0].as_h + width = thumbnail["width"].as_i + height = thumbnail["height"].as_i + aspect_ratio = (width.to_f / height.to_f) + url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") + json.field "image" do + json.array do + IMAGE_QUALITIES.each do |quality| + json.object do + json.field "url", url.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", (quality / aspect_ratio).ceil.to_i + end end end end @@ -202,137 +205,137 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing end end end - end - when .has_key?("postMultiImageRenderer") - json.object do - attachment = attachment["postMultiImageRenderer"] - json.field "type", "multiImage" - json.field "images" do - json.array do - attachment["images"].as_a.each do |image| - json.array do - thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h - width = thumbnail["width"].as_i - height = thumbnail["height"].as_i - aspect_ratio = (width.to_f / height.to_f) - url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") + when .has_key?("postMultiImageRenderer") + json.object do + attachment = attachment["postMultiImageRenderer"] + json.field "type", "multiImage" + json.field "images" do + json.array do + attachment["images"].as_a.each do |image| + json.array do + thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h + width = thumbnail["width"].as_i + height = thumbnail["height"].as_i + aspect_ratio = (width.to_f / height.to_f) + url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - IMAGE_QUALITIES.each do |quality| - json.object do - json.field "url", url.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", (quality / aspect_ratio).ceil.to_i + IMAGE_QUALITIES.each do |quality| + json.object do + json.field "url", url.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", (quality / aspect_ratio).ceil.to_i + end end end end end end end - end - when .has_key?("playlistRenderer") - parse_item(attachment) - .as(SearchPlaylist) - .to_json(locale, json) - when .has_key?("quizRenderer") - json.object do - attachment = attachment["quizRenderer"] - json.field "type", "quiz" - json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) - json.field "choices" do - json.array do - attachment["choices"].as_a.each do |choice| - json.object do - json.field "text", choice.dig("text", "runs", 0, "text").as_s - json.field "isCorrect", choice["isCorrect"].as_bool + when .has_key?("playlistRenderer") + parse_item(attachment) + .as(SearchPlaylist) + .to_json(locale, json) + when .has_key?("quizRenderer") + json.object do + attachment = attachment["quizRenderer"] + json.field "type", "quiz" + json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) + json.field "choices" do + json.array do + attachment["choices"].as_a.each do |choice| + json.object do + json.field "text", choice.dig("text", "runs", 0, "text").as_s + json.field "isCorrect", choice["isCorrect"].as_bool + end end end end end - end - else - json.object do - json.field "type", "unknown" - json.field "error", "Unrecognized attachment type." + else + json.object do + json.field "type", "unknown" + json.field "error", "Unrecognized attachment type." + end end end end - end - if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || - comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) - .try &.as_s.gsub(/\D/, "").to_i?) - continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s - continuation ||= "" + if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || + comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) + .try &.as_s.gsub(/\D/, "").to_i?) + continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s + continuation ||= "" - json.field "replies" do - json.object do - json.field "replyCount", reply_count - json.field "continuation", extract_channel_community_cursor(continuation) + json.field "replies" do + json.object do + json.field "replyCount", reply_count + json.field "continuation", extract_channel_community_cursor(continuation) + end end end end end end end - end - if !is_single_post - if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") - json.field "continuation", extract_channel_community_cursor(cont.as_s) + if !is_single_post + if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + json.field "continuation", extract_channel_community_cursor(cont.as_s) + end end end end - end - if format == "html" - response = JSON.parse(response) - content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode) + if format == "html" + response = JSON.parse(response) + content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode) - response = JSON.build do |json| - json.object do - json.field "contentHtml", content_html + response = JSON.build do |json| + json.object do + json.field "contentHtml", content_html + end end end + + return response end - return response -end + def produce_channel_community_continuation(ucid, cursor) + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => cursor || "", + }, + } -def produce_channel_community_continuation(ucid, cursor) - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => cursor || "", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def extract_channel_community_cursor(continuation) - object = URI.decode_www_form(continuation) - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } - .try(&.["80226972:0:embedded"]["3:1:base64"].as_h) - - if object["53:2:embedded"]?.try &.["3:0:embedded"]? - object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] - .try(&.["2:0:base64"].as_h) - .try { |i| Protodec::Any.cast_json(i) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i, padding: false) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } - object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") + return continuation end - cursor = Protodec::Any.cast_json(object) - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } + def extract_channel_community_cursor(continuation) + object = URI.decode_www_form(continuation) + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + .try(&.["80226972:0:embedded"]["3:1:base64"].as_h) - cursor + if object["53:2:embedded"]?.try &.["3:0:embedded"]? + object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"] + .try(&.["2:0:base64"].as_h) + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i, padding: false) } + + object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") + end + + cursor = Protodec::Any.cast_json(object) + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + + cursor + end end diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr index cba1abd9c..d7491873a 100644 --- a/src/invidious/channels/playlists.cr +++ b/src/invidious/channels/playlists.cr @@ -1,55 +1,59 @@ -def fetch_channel_playlists(ucid, author, continuation, sort_by) - if continuation - initial_data = YoutubeAPI.browse(continuation) - else - params = - case sort_by - when "last", "last_added" - # Equivalent to "&sort=lad" - # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} - "EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA" - when "oldest", "oldest_created" - # formerly "&sort=da" - # Not available anymore :c or maybe ?? - # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} - "EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA" - # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} - # "EglwbGF5bGlzdHMYASABMAE%3D" - when "newest", "newest_created" - # Formerly "&sort=dd" - # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} - "EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA" - end +module Invidious::Channels::Playlists + extend self - initial_data = YoutubeAPI.browse(ucid, params: params || "") + def fetch_channel_playlists(ucid, author, continuation, sort_by) + if continuation + initial_data = YoutubeAPI.browse(continuation) + else + params = + case sort_by + when "last", "last_added" + # Equivalent to "&sort=lad" + # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA" + when "oldest", "oldest_created" + # formerly "&sort=da" + # Not available anymore :c or maybe ?? + # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA" + # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} + # "EglwbGF5bGlzdHMYASABMAE%3D" + when "newest", "newest_created" + # Formerly "&sort=dd" + # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}} + "EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA" + end + + initial_data = YoutubeAPI.browse(ucid, params: params || "") + end + + return extract_items(initial_data, author, ucid) end - return extract_items(initial_data, author, ucid) -end - -def fetch_channel_podcasts(ucid, author, continuation) - if continuation - initial_data = YoutubeAPI.browse(continuation) - else - initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA") + def fetch_channel_podcasts(ucid, author, continuation) + if continuation + initial_data = YoutubeAPI.browse(continuation) + else + initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA") + end + return extract_items(initial_data, author, ucid) end - return extract_items(initial_data, author, ucid) -end -def fetch_channel_releases(ucid, author, continuation) - if continuation - initial_data = YoutubeAPI.browse(continuation) - else - initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA") + def fetch_channel_releases(ucid, author, continuation) + if continuation + initial_data = YoutubeAPI.browse(continuation) + else + initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA") + end + return extract_items(initial_data, author, ucid) end - return extract_items(initial_data, author, ucid) -end -def fetch_channel_courses(ucid, author, continuation) - if continuation - initial_data = YoutubeAPI.browse(continuation) - else - initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D") + def fetch_channel_courses(ucid, author, continuation) + if continuation + initial_data = YoutubeAPI.browse(continuation) + else + initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D") + end + return extract_items(initial_data, author, ucid) end - return extract_items(initial_data, author, ucid) end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/tabs.cr similarity index 100% rename from src/invidious/channels/videos.cr rename to src/invidious/channels/tabs.cr diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index 134a3d712..d2679aee5 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -246,7 +246,7 @@ module Invidious::Comments end content_html = html_content || "" - json.field "content", Helpers.html_to_content(content_html) + json.field "content", Invidious::Helpers.html_to_content(content_html) json.field "contentHtml", content_html if published_text != nil diff --git a/src/invidious/ext/kemal_static_file_handler.cr b/src/invidious/ext/kemal_static_file_handler.cr new file mode 100644 index 000000000..16cb84fbf --- /dev/null +++ b/src/invidious/ext/kemal_static_file_handler.cr @@ -0,0 +1,216 @@ +{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %} + # Strip StaticFileHandler from the binary + # + # This allows us to compile on 1.17.0 as the compiler won't try to + # semantically check the outdated upstream code. + class Kemal::Config + private def setup_static_file_handler + end + end + + # Nullify `Kemal::StaticFileHandler` + # + # Needed until the next release of Kemal after 1.7 + class Kemal::StaticFileHandler < HTTP::StaticFileHandler + def call(context : HTTP::Server::Context) + end + end + + {% skip_file %} +{% end %} + +# Since systems have a limit on number of open files (`ulimit -a`), +# we serve them from memory to avoid 'Too many open files' without needing +# to modify ulimit. +# +# Very heavily re-used: +# https://github.com/kemalcr/kemal/blob/master/src/kemal/helpers/helpers.cr +# https://github.com/kemalcr/kemal/blob/master/src/kemal/static_file_handler.cr +# +# Changes: +# - A `send_file` overload is added which supports sending a Slice, file_path, filestat +# - `StaticFileHandler` is patched to cache to and serve from @cached_files + +private def multipart(file, env : HTTP::Server::Context) + # See http://httpwg.org/specs/rfc7233.html + fileb = file.size + startb = endb = 0 + + if match = env.request.headers["Range"].match /bytes=(\d{1,})-(\d{0,})/ + startb = match[1].to_i { 0 } if match.size >= 2 + endb = match[2].to_i { 0 } if match.size >= 3 + end + + endb = fileb - 1 if endb == 0 + + if startb < endb < fileb + content_length = 1 + endb - startb + env.response.status_code = 206 + env.response.content_length = content_length + env.response.headers["Accept-Ranges"] = "bytes" + env.response.headers["Content-Range"] = "bytes #{startb}-#{endb}/#{fileb}" # MUST + + if startb > 1024 + skipped = 0 + # file.skip only accepts values less or equal to 1024 (buffer size, undocumented) + until (increase_skipped = skipped + 1024) > startb + file.skip(1024) + skipped = increase_skipped + end + if (skipped_minus_startb = skipped - startb) > 0 + file.skip skipped_minus_startb + end + else + file.skip(startb) + end + + IO.copy(file, env.response, content_length) + else + env.response.content_length = fileb + env.response.status_code = 200 # Range not satisfable, see 4.4 Note + IO.copy(file, env.response) + end +end + +# Set the Content-Disposition to "attachment" with the specified filename, +# instructing the user agents to prompt to save. +private def attachment(env : HTTP::Server::Context, filename : String? = nil, disposition : String? = nil) + disposition = "attachment" if disposition.nil? && filename + if disposition && filename + env.response.headers["Content-Disposition"] = "#{disposition}; filename=\"#{File.basename(filename)}\"" + end +end + +def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt8), filestat : File::Info, filename : String? = nil, disposition : String? = nil) + config = Kemal.config.serve_static + mime_type = MIME.from_filename(file_path, "application/octet-stream") + env.response.content_type = mime_type + env.response.headers["Accept-Ranges"] = "bytes" + env.response.headers["X-Content-Type-Options"] = "nosniff" + minsize = 860 # http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits ?? + request_headers = env.request.headers + filesize = data.bytesize + attachment(env, filename, disposition) + + Kemal.config.static_headers.try(&.call(env, file_path, filestat)) + + file = IO::Memory.new(data) + if env.request.method == "GET" && env.request.headers.has_key?("Range") + return multipart(file, env) + end + + condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path) + if condition && request_headers.includes_word?("Accept-Encoding", "gzip") + env.response.headers["Content-Encoding"] = "gzip" + Compress::Gzip::Writer.open(env.response) do |deflate| + IO.copy(file, deflate) + end + elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate") + env.response.headers["Content-Encoding"] = "deflate" + Compress::Deflate::Writer.open(env.response) do |deflate| + IO.copy(file, deflate) + end + else + env.response.content_length = filesize + IO.copy(file, env.response) + end + + return +end + +module Kemal + class StaticFileHandler < HTTP::StaticFileHandler + CACHE_LIMIT = 5_000_000 # 5MB + @cached_files = {} of String => {data: Bytes, filestat: File::Info} + + def call(context : HTTP::Server::Context) + return call_next(context) if context.request.path.not_nil! == "/" + + case context.request.method + when "GET", "HEAD" + else + if @fallthrough + call_next(context) + else + context.response.status = HTTP::Status::METHOD_NOT_ALLOWED + context.response.headers.add("Allow", "GET, HEAD") + end + return + end + + config = Kemal.config.serve_static + original_path = context.request.path.not_nil! + request_path = URI.decode_www_form(original_path) + + # File path cannot contains '\0' (NUL) because all filesystem I know + # don't accept '\0' character as file name. + if request_path.includes? '\0' + context.response.status = HTTP::Status::BAD_REQUEST + return + end + + expanded_path = File.expand_path(request_path, "/") + is_dir_path = if original_path.ends_with?('/') && !expanded_path.ends_with? '/' + expanded_path = expanded_path + '/' + true + else + expanded_path.ends_with? '/' + end + + file_path = File.join(@public_dir, expanded_path) + + if file = @cached_files[file_path]? + last_modified = file[:filestat].modification_time + add_cache_headers(context.response.headers, last_modified) + + if cache_request?(context, last_modified) + context.response.status = HTTP::Status::NOT_MODIFIED + return + end + + send_file(context, file_path, file[:data], file[:filestat]) + else + file_info = File.info?(file_path) + is_dir = file_info.try &.directory? || false + is_file = file_info.try &.file? || false + + if request_path != expanded_path + redirect_to context, expanded_path + elsif is_dir && !is_dir_path + redirect_to context, expanded_path + '/' + end + + return call_next(context) if file_info.nil? + + if is_dir + if config.is_a?(Hash) && config["dir_listing"] == true + context.response.content_type = "text/html" + directory_listing(context.response, request_path, file_path) + else + call_next(context) + end + elsif is_file + last_modified = file_info.modification_time + add_cache_headers(context.response.headers, last_modified) + + if cache_request?(context, last_modified) + context.response.status = HTTP::Status::NOT_MODIFIED + return + end + + if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT + data = Bytes.new(size) + File.open(file_path, &.read(data)) + + @cached_files[file_path] = {data: data, filestat: file_info} + send_file(context, file_path, data, file_info) + else + send_file(context, file_path) + end + else # Not a normal file (FIFO/device/socket) + call_next(context) + end + end + end + end +end diff --git a/src/invidious/feeds/trending.cr b/src/invidious/feeds/trending.cr new file mode 100644 index 000000000..41c2eb330 --- /dev/null +++ b/src/invidious/feeds/trending.cr @@ -0,0 +1,52 @@ +module Invidious::Feeds::Trending + extend self + + def fetch(trending_type, region, locale) + region ||= "US" + region = region.upcase + + plid = nil + + browse_id = "" + + case trending_type.try &.downcase + when "gaming" + browse_id = "UCOpNcN46UbXVtpKMrmU4Abg" + params = "Egh0cmVuZGluZw%3D%3D" + when "livestreams" + browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" + params = "EgdsaXZldGFikgEDCKEK" + else + # Livestreams is the default one as Youtube removed + # the aggregated trending page + # https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458 + browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" + params = "EgdsaXZldGFikgEDCKEK" + end + + client_config = YoutubeAPI::ClientConfig.new(region: region) + initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config) + + items, _ = extract_items(initial_data) + + extracted = [] of SearchItem + + deduplicate = items.size > 1 + + items.each do |itm| + if itm.is_a?(Category) + # Ignore the smaller categories, as they generally contain a sponsored + # channel, which brings a lot of noise on the trending page. + # See: https://github.com/iv-org/invidious/issues/2989 + next if (itm.contents.size < 24 && deduplicate) + + extracted.concat itm.contents.select(SearchItem) + else + extracted << itm + end + end + + # Deduplicate items before returning results + return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid + end +end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index cee6eb2c8..7ab92314c 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -2,26 +2,29 @@ # Issue template # ------------------- -macro error_template(*args) - error_template_helper(env, {{args.splat}}) -end +module Errors + extend self -def github_details(summary : String, content : String) - details = %(\n
) - details += %(\n#{summary}) - details += %(\n

) - details += %(\n \n```\n) - details += content.strip - details += %(\n```) - details += %(\n

) - details += %(\n
) - return HTML.escape(details) -end + macro error_template(*args) + Errors.error_template_helper(env, {{args.splat}}) + end -def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String) - issue_title = "#{exception.message} (#{exception.class})" + def github_details(summary : String, content : String) + details = %(\n
) + details += %(\n#{summary}) + details += %(\n

) + details += %(\n \n```\n) + details += content.strip + details += %(\n```) + details += %(\n

) + details += %(\n
) + return HTML.escape(details) + end - issue_template = <<-TEXT + def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String) + issue_title = "#{exception.message} (#{exception.class})" + + issue_template = <<-TEXT Title: `#{HTML.escape(issue_title)}` Date: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}` Route: `#{HTML.escape(env.request.resource)}` @@ -29,39 +32,39 @@ def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tup TEXT - issue_template += github_details("Backtrace", exception.inspect_with_backtrace) + issue_template += github_details("Backtrace", exception.inspect_with_backtrace) - return issue_title, issue_template -end - -def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) - if exception.is_a?(InfoException) - return error_template_helper(env, status_code, exception.message || "") + return issue_title, issue_template end - locale = env.get("preferences").as(Preferences).locale + def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_template_helper(env, status_code, exception.message || "") + end - env.response.content_type = "text/html" - env.response.status_code = status_code + locale = env.get("preferences").as(Preferences).locale - # Unpacking into issue_title, issue_template directly causes a compiler error - # I have no idea why. - issue_template_components = get_issue_template(env, exception) - issue_title, issue_template = issue_template_components + env.response.content_type = "text/html" + env.response.status_code = status_code - # URLs for the error message below - url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" - url_search_issues = "https://github.com/iv-org/invidious/issues" - url_search_issues += "?q=is:issue+is:open+" - url_search_issues += URI.encode_www_form("[Bug] #{issue_title}") + # Unpacking into issue_title, issue_template directly causes a compiler error + # I have no idea why. + issue_template_components = self.get_issue_template(env, exception) + issue_title, issue_template = issue_template_components - url_switch = "https://redirect.invidious.io" + env.request.resource + # URLs for the error message below + url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" + url_search_issues = "https://github.com/iv-org/invidious/issues" + url_search_issues += "?q=is:issue+is:open+" + url_search_issues += URI.encode_www_form("[Bug] #{issue_title}") - url_new_issue = "https://github.com/iv-org/invidious/issues/new" - url_new_issue += "?labels=bug&template=bug_report.md&title=" - url_new_issue += URI.encode_www_form("[Bug] " + issue_title) + url_switch = "https://redirect.invidious.io" + env.request.resource - error_message = <<-END_HTML + url_new_issue = "https://github.com/iv-org/invidious/issues/new" + url_new_issue += "?labels=bug&template=bug_report.md&title=" + url_new_issue += URI.encode_www_form("[Bug] " + issue_title) + + error_message = <<-END_HTML

#{I18n.translate(locale, "crash_page_you_found_a_bug")}



@@ -82,116 +85,116 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
END_HTML - # Don't show the usual "next steps" widget. The same options are - # proposed above the error message, just worded differently. - next_steps = "" + # Don't show the usual "next steps" widget. The same options are + # proposed above the error message, just worded differently. + next_steps = "" - return templated "error" -end - -def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String) - env.response.content_type = "text/html" - env.response.status_code = status_code - - locale = env.get("preferences").as(Preferences).locale - - error_message = I18n.translate(locale, message) - next_steps = error_redirect_helper(env) - - return templated "error" -end - -# ------------------- -# Atom feeds -# ------------------- - -macro error_atom(*args) - error_atom_helper(env, {{args.splat}}) -end - -def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) - if exception.is_a?(InfoException) - return error_atom_helper(env, status_code, exception.message || "") + return templated "error" end - env.response.content_type = "application/atom+xml" - env.response.status_code = status_code + def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String) + env.response.content_type = "text/html" + env.response.status_code = status_code - return "#{exception.inspect_with_backtrace}" -end + locale = env.get("preferences").as(Preferences).locale -def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String) - env.response.content_type = "application/atom+xml" - env.response.status_code = status_code + error_message = I18n.translate(locale, message) + next_steps = error_redirect_helper(env) - return "#{message}" -end - -# ------------------- -# JSON -# ------------------- - -macro error_json(*args) - error_json_helper(env, {{args.splat}}) -end - -def error_json_helper( - env : HTTP::Server::Context, - status_code : Int32, - exception : Exception, - additional_fields : Hash(String, Object) | Nil = nil, -) - if exception.is_a?(InfoException) - return error_json_helper(env, status_code, exception.message || "", additional_fields) + return templated "error" end - env.response.content_type = "application/json" - env.response.status_code = status_code + # ------------------- + # Atom feeds + # ------------------- - error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} - - if additional_fields - error_message = error_message.merge(additional_fields) + macro error_atom(*args) + Errors.error_atom_helper(env, {{args.splat}}) end - return error_message.to_json -end + def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_atom_helper(env, status_code, exception.message || "") + end -def error_json_helper( - env : HTTP::Server::Context, - status_code : Int32, - message : String, - additional_fields : Hash(String, Object) | Nil = nil, -) - env.response.content_type = "application/json" - env.response.status_code = status_code + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code - error_message = {"error" => message} - - if additional_fields - error_message = error_message.merge(additional_fields) + return "#{exception.inspect_with_backtrace}" end - return error_message.to_json -end + def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String) + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code -# ------------------- -# Redirect -# ------------------- + return "#{message}" + end -def error_redirect_helper(env : HTTP::Server::Context) - request_path = env.request.path + # ------------------- + # JSON + # ------------------- - locale = env.get("preferences").as(Preferences).locale + macro error_json(*args) + Errors.error_json_helper(env, {{args.splat}}) + end - if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || - request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL") - next_steps_text = I18n.translate(locale, "next_steps_error_message") - refresh = I18n.translate(locale, "next_steps_error_message_refresh") - go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube") - switch_instance = I18n.translate(locale, "Switch Invidious Instance") + def error_json_helper( + env : HTTP::Server::Context, + status_code : Int32, + exception : Exception, + additional_fields : Hash(String, Object) | Nil = nil, + ) + if exception.is_a?(InfoException) + return error_json_helper(env, status_code, exception.message || "", additional_fields) + end - return <<-END_HTML + env.response.content_type = "application/json" + env.response.status_code = status_code + + error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} + + if additional_fields + error_message = error_message.merge(additional_fields) + end + + return error_message.to_json + end + + def error_json_helper( + env : HTTP::Server::Context, + status_code : Int32, + message : String, + additional_fields : Hash(String, Object) | Nil = nil, + ) + env.response.content_type = "application/json" + env.response.status_code = status_code + + error_message = {"error" => message} + + if additional_fields + error_message = error_message.merge(additional_fields) + end + + return error_message.to_json + end + + # ------------------- + # Redirect + # ------------------- + + def error_redirect_helper(env : HTTP::Server::Context) + request_path = env.request.path + + locale = env.get("preferences").as(Preferences).locale + + if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || + request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL") + next_steps_text = I18n.translate(locale, "next_steps_error_message") + refresh = I18n.translate(locale, "next_steps_error_message_refresh") + go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube") + switch_instance = I18n.translate(locale, "Switch Invidious Instance") + + return <<-END_HTML

#{next_steps_text}

END_HTML - else - return "" + else + return "" + end end end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 8395c7a4d..377ba4194 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -83,7 +83,7 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil) + scopes, _, _ = Invidious::Helpers::Tokens.validate_request(token, session, env.request, HMAC_KEY, nil) if email = Invidious::Database::SessionIDs.select_email(session) user = Invidious::Database::Users.select!(email: email) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index a08135eaa..c987fc6ec 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -22,7 +22,7 @@ struct Annotation property annotations : String end -module Helpers +module Invidious::Helpers extend self private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} @@ -62,7 +62,7 @@ module Helpers end def create_notification_stream(env, topics, connection_channel) - connection = Channel(PQ::Notification).new(8) + connection = ::Channel(PQ::Notification).new(8) connection_channel.send({true, connection}) locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 17dff5664..32eae12bd 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -53,7 +53,7 @@ struct SearchVideo xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") end - xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) } + xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Invidious::Helpers.html_to_content(self.description_html) } end end @@ -63,7 +63,7 @@ struct SearchVideo xml.element("media:title") { xml.text self.title } xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") - xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) } + xml.element("media:description") { xml.text Invidious::Helpers.html_to_content(self.description_html) } end xml.element("media:community") do @@ -111,7 +111,7 @@ struct SearchVideo Invidious::JSONify::APIv1.thumbnails(json, self.id) end - json.field "description", Helpers.html_to_content(self.description_html) + json.field "description", Invidious::Helpers.html_to_content(self.description_html) json.field "descriptionHtml", self.description_html json.field "viewCount", self.views @@ -255,7 +255,7 @@ struct SearchChannel json.field "videoCount", self.video_count json.field "channelHandle", self.channel_handle - json.field "description", Helpers.html_to_content(self.description_html) + json.field "description", Invidious::Helpers.html_to_content(self.description_html) json.field "descriptionHtml", self.description_html end end @@ -332,7 +332,7 @@ struct ProblematicTimelineItem end xml.element("pre") do - get_issue_template(env, @parse_exception) + Errors.get_issue_template(env, @parse_exception) end end end diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index a44988cd3..623937f8c 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,145 +1,149 @@ require "crypto/subtle" -def generate_token(email, scopes, expire, key) - session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" - Invidious::Database::SessionIDs.insert(session, email) +module Invidious::Helpers::Tokens + extend self - token = { - "session" => session, - "scopes" => scopes, - "expire" => expire, - } + def generate_token(email, scopes, expire, key) + session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" + Invidious::Database::SessionIDs.insert(session, email) - if !expire - token.delete("expire") - end + token = { + "session" => session, + "scopes" => scopes, + "expire" => expire, + } - token["signature"] = sign_token(key, token) - - return token.to_json -end - -def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false) - expire = Time.utc + expire - - token = { - "session" => session, - "expire" => expire.to_unix, - "scopes" => scopes, - } - - if use_nonce - nonce = Random::Secure.hex(16) - Invidious::Database::Nonces.insert(nonce, expire) - token["nonce"] = nonce - end - - token["signature"] = sign_token(key, token) - - return token.to_json -end - -def sign_token(key, hash) - string_to_sign = [] of String - - # TODO: figure out which "key" variable is used - # Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this - # variable, but it's preferable to not touch that (works fine atm). - hash.each do |key, value| - next if key == "signature" - - if value.is_a?(JSON::Any) && value.as_a? - value = value.as_a.map(&.as_s) + if !expire + token.delete("expire") end - case value - when Array - string_to_sign << "#{key}=#{value.sort.join(",")}" - when Tuple - string_to_sign << "#{key}=#{value.to_a.sort.join(",")}" - else - string_to_sign << "#{key}=#{value}" + token["signature"] = sign_token(key, token) + + return token.to_json + end + + def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false) + expire = Time.utc + expire + + token = { + "session" => session, + "expire" => expire.to_unix, + "scopes" => scopes, + } + + if use_nonce + nonce = Random::Secure.hex(16) + Invidious::Database::Nonces.insert(nonce, expire) + token["nonce"] = nonce end + + token["signature"] = sign_token(key, token) + + return token.to_json end - string_to_sign = string_to_sign.sort.join("\n") - return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip -end + def sign_token(key, hash) + string_to_sign = [] of String -def validate_request(token, session, request, key, locale = nil) - case token - when String - token = JSON.parse(URI.decode_www_form(token)).as_h - when JSON::Any - token = token.as_h - when Nil - raise InfoException.new("Hidden field \"token\" is a required field") + # TODO: figure out which "key" variable is used + # Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this + # variable, but it's preferable to not touch that (works fine atm). + hash.each do |key, value| + next if key == "signature" + + if value.is_a?(JSON::Any) && value.as_a? + value = value.as_a.map(&.as_s) + end + + case value + when Array + string_to_sign << "#{key}=#{value.sort.join(",")}" + when Tuple + string_to_sign << "#{key}=#{value.to_a.sort.join(",")}" + else + string_to_sign << "#{key}=#{value}" + end + end + + string_to_sign = string_to_sign.sort.join("\n") + return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip end - expire = token["expire"]?.try &.as_i - if expire.try &.< Time.utc.to_unix - raise InfoException.new("Token is expired, please try again") - end + def validate_request(token, session, request, key, locale = nil) + case token + when String + token = JSON.parse(URI.decode_www_form(token)).as_h + when JSON::Any + token = token.as_h + when Nil + raise InfoException.new("Hidden field \"token\" is a required field") + end - if token["session"] != session - raise InfoException.new("Erroneous token") - end + expire = token["expire"]?.try &.as_i + if expire.try &.< Time.utc.to_unix + raise InfoException.new("Token is expired, please try again") + end - scopes = token["scopes"].as_a.map(&.as_s) - scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" - if !scopes_include_scope(scopes, scope) - raise InfoException.new("Invalid scope") - end - - if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token)) - raise InfoException.new("Invalid signature") - end - - if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s)) - if nonce[1] > Time.utc - Invidious::Database::Nonces.update_set_expired(nonce[0]) - else + if token["session"] != session raise InfoException.new("Erroneous token") end - end - return {scopes, expire, token["signature"].as_s} -end - -def scope_includes_scope(scope, subset) - methods, endpoint = scope.split(":") - methods = methods.split(";").map(&.upcase).reject(&.empty?).sort! - endpoint = endpoint.downcase - - subset_methods, subset_endpoint = subset.split(":") - subset_methods = subset_methods.split(";").map(&.upcase).sort! - subset_endpoint = subset_endpoint.downcase - - if methods.empty? - methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS) - end - - if methods & subset_methods != subset_methods - return false - end - - if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*") - return false - end - - if !endpoint.ends_with?("*") && subset_endpoint != endpoint - return false - end - - return true -end - -def scopes_include_scope(scopes, subset) - scopes.each do |scope| - if scope_includes_scope(scope, subset) - return true + scopes = token["scopes"].as_a.map(&.as_s) + scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" + if !scopes_include_scope(scopes, scope) + raise InfoException.new("Invalid scope") end + + if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token)) + raise InfoException.new("Invalid signature") + end + + if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s)) + if nonce[1] > Time.utc + Invidious::Database::Nonces.update_set_expired(nonce[0]) + else + raise InfoException.new("Erroneous token") + end + end + + return {scopes, expire, token["signature"].as_s} end - return false + def scope_includes_scope(scope, subset) + methods, endpoint = scope.split(":") + methods = methods.split(";").map(&.upcase).reject(&.empty?).sort! + endpoint = endpoint.downcase + + subset_methods, subset_endpoint = subset.split(":") + subset_methods = subset_methods.split(";").map(&.upcase).sort! + subset_endpoint = subset_endpoint.downcase + + if methods.empty? + methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS) + end + + if methods & subset_methods != subset_methods + return false + end + + if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*") + return false + end + + if !endpoint.ends_with?("*") && subset_endpoint != endpoint + return false + end + + return true + end + + def scopes_include_scope(scopes, subset) + scopes.each do |scope| + if self.scope_includes_scope(scope, subset) + return true + end + end + + return false + end end diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 5c5ecc837..43374f46b 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh) + channel = Invidious::Channels::Channels.fetch_channel(id, pull_all_videos: CONFIG.full_refresh) lim_fibers = max_fibers diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 4f8130dfd..dfb93ebf4 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -54,7 +54,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob # While iterating through, we may have an email stored from a deleted account if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) LOGGER.info("RefreshFeedsJob: CREATE #{view_name}") - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") + db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(email)}") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr deleted file mode 100644 index e6a06633a..000000000 --- a/src/invidious/playlists.cr +++ /dev/null @@ -1,579 +0,0 @@ -struct PlaylistVideo - include DB::Serializable - - property title : String - property id : String - property author : String - property ucid : String - property length_seconds : Int32 - property published : Time - property plid : String - property index : Int64 - property live_now : Bool - - def to_xml(xml : XML::Builder) - xml.element("entry") do - xml.element("id") { xml.text "yt:video:#{self.id}" } - xml.element("yt:videoId") { xml.text self.id } - xml.element("yt:channelId") { xml.text self.ucid } - xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") - - xml.element("author") do - xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } - end - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do - xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") - end - end - end - - xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } - - xml.element("media:group") do - xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", - width: "320", height: "180") - end - end - end - - def to_xml(_xml : Nil = nil) - XML.build { |xml| to_xml(xml) } - end - - def to_json(locale : String?, json : JSON::Builder) - to_json(json) - end - - def to_json(json : JSON::Builder, index : Int32? = nil) - json.object do - json.field "type", "video" - - json.field "title", self.title - json.field "videoId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do - Invidious::JSONify::APIv1.thumbnails(json, self.id) - end - - if index - json.field "index", index - json.field "indexId", self.index.to_u64.to_s(16).upcase - else - json.field "index", self.index - end - - json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now - end - end - - def to_json(_json : Nil, index : Int32? = nil) - JSON.build { |json| to_json(json, index: index) } - end -end - -struct Playlist - include DB::Serializable - - property title : String - property id : String - property author : String - property author_thumbnail : String - property ucid : String - property description : String - property description_html : String - property video_count : Int32 - property views : Int64 - property updated : Time - property thumbnail : String? - property subtitle : String? - - def to_json(offset, json : JSON::Builder, video_id : String? = nil) - json.object do - json.field "type", "playlist" - json.field "title", self.title - json.field "playlistId", self.id - json.field "playlistThumbnail", self.thumbnail - - json.field "author", self.author - json.field "authorId", self.ucid - if !self.ucid.empty? - json.field "authorUrl", "/channel/#{self.ucid}" - else - json.field "authorUrl", "" - end - json.field "subtitle", self.subtitle - - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - - json.field "description", self.description - json.field "descriptionHtml", self.description_html - json.field "videoCount", self.video_count - - json.field "viewCount", self.views - json.field "updated", self.updated.to_unix - json.field "isListed", self.privacy.public? - - json.field "videos" do - json.array do - videos = get_playlist_videos(self, offset: offset, video_id: video_id) - videos.each do |video| - video.to_json(json) - end - end - end - end - end - - def to_json(offset, _json : Nil = nil, video_id : String? = nil) - JSON.build do |json| - to_json(offset, json, video_id: video_id) - end - end - - def privacy - PlaylistPrivacy::Public - end -end - -enum PlaylistPrivacy - Public = 0 - Unlisted = 1 - Private = 2 -end - -struct InvidiousPlaylist - include DB::Serializable - - property title : String - property id : String - property author : String - property description : String = "" - property video_count : Int32 - property created : Time - property updated : Time - - @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)] - property privacy : PlaylistPrivacy = PlaylistPrivacy::Private - property index : Array(Int64) - - @[DB::Field(ignore: true)] - property thumbnail_id : String? - - module PlaylistPrivacyConverter - def self.from_rs(rs) - return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) - end - end - - def to_json(offset, json : JSON::Builder, video_id : String? = nil) - json.object do - json.field "type", "invidiousPlaylist" - json.field "title", self.title - json.field "playlistId", self.id - - json.field "author", self.author - json.field "authorId", self.ucid - json.field "authorUrl", nil - json.field "authorThumbnails", [] of String - - json.field "description", Helpers.html_to_content(self.description_html) - json.field "descriptionHtml", self.description_html - json.field "videoCount", self.video_count - - json.field "viewCount", self.views - json.field "updated", self.updated.to_unix - json.field "isListed", self.privacy.public? - - json.field "videos" do - json.array do - if (!offset || offset == 0) && !video_id.nil? - index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id) - offset = self.index.index(index) || 0 - end - - videos = get_playlist_videos(self, offset: offset, video_id: video_id) - videos.each_with_index do |video, idx| - video.to_json(json, offset + idx) - end - end - end - end - end - - def to_json(offset, _json : Nil = nil, video_id : String? = nil) - JSON.build do |json| - to_json(offset, json, video_id: video_id) - end - end - - def thumbnail - # TODO: Get playlist thumbnail from playlist data rather than first video - @thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------" - "/vi/#{@thumbnail_id}/mqdefault.jpg" - end - - def author_thumbnail - nil - end - - def ucid - nil - end - - def views - 0_i64 - end - - def description_html - HTML.escape(self.description) - end -end - -def create_playlist(title, privacy, user) - plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" - - playlist = InvidiousPlaylist.new({ - title: title.byte_slice(0, 150), - id: plid, - author: user.email, - description: "", # Max 5000 characters - video_count: 0, - created: Time.utc, - updated: Time.utc, - privacy: privacy, - index: [] of Int64, - }) - - Invidious::Database::Playlists.insert(playlist) - - return playlist -end - -def subscribe_playlist(user, playlist) - playlist = InvidiousPlaylist.new({ - title: playlist.title[..150], - id: playlist.id, - author: user.email, - description: "", # Max 5000 characters - video_count: playlist.video_count, - created: Time.utc, - updated: playlist.updated, - privacy: PlaylistPrivacy::Private, - index: [] of Int64, - }) - - Invidious::Database::Playlists.insert(playlist) - - return playlist -end - -def produce_playlist_continuation(id, index) - if id.starts_with? "UC" - id = "UU" + id.lchop("UC") - end - plid = "VL" + id - - # Emulate a "request counter" increment, to make perfectly valid - # ctokens, even if at the time of writing, it's ignored by youtube. - request_count = (index / 100).to_i64 || 1_i64 - - data = {"1:varint" => index.to_i64} - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i, padding: false) } - - object = { - "80226972:embedded" => { - "2:string" => plid, - "3:base64" => { - "1:varint" => request_count, - "15:string" => "PT:#{data}", - "104:embedded" => {"1:0:varint" => 0_i64}, - }, - "35:string" => id, - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def get_playlist(plid : String) - if plid.starts_with? "IV" - if playlist = Invidious::Database::Playlists.select(id: plid) - return playlist - else - raise NotFoundException.new("Playlist does not exist.") - end - else - return fetch_playlist(plid) - end -end - -def fetch_playlist(plid : String) - if plid.starts_with? "UC" - plid = "UU#{plid.lchop("UC")}" - end - - initial_data = YoutubeAPI.browse("VL" + plid, params: "") - - playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items") - raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer - - playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer") - raise InfoException.new("Could not extract playlist info") if !playlist_info - - title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || "" - - desc_item = playlist_info["description"]? - - description_txt = desc_item.try &.["runs"]?.try &.as_a - .map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" - - description_html = desc_item.try &.["runs"]?.try &.as_a - .try { |run| content_to_comment_html(run).try &.to_s } || "

" - - thumbnail = playlist_info.dig?( - "thumbnailRenderer", "playlistVideoThumbnailRenderer", - "thumbnail", "thumbnails", 0, "url" - ).try &.as_s || playlist_info.dig?( - "thumbnailRenderer", "playlistCustomThumbnailRenderer", - "thumbnail", "thumbnails", 0, "url" - ).try &.as_s - - views = 0_i64 - updated = Time.utc - video_count = 0 - - subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle")) - - playlist_info["stats"]?.try &.as_a.each do |stat| - text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s - next if !text - - if text.includes? "video" - video_count = text.gsub(/\D/, "").to_i? || 0 - elsif text.includes? "episode" - video_count = text.gsub(/\D/, "").to_i? || 0 - elsif text.includes? "view" - views = text.gsub(/\D/, "").to_i64? || 0_i64 - elsif !text.includes? "Pay to watch" - updated = decode_date(text.lchop("Last updated on ").lchop("Updated ")) - end - end - - if playlist_sidebar_renderer.size < 2 - author = "" - author_thumbnail = "" - ucid = "" - else - author_info = playlist_sidebar_renderer[1].dig?( - "playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer" - ) - - raise InfoException.new("Could not extract author info") if !author_info - - author = author_info.dig?("title", "runs", 0, "text").try &.as_s || "" - author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || "" - ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" - end - - return Playlist.new({ - title: title, - id: plid, - author: author, - author_thumbnail: author_thumbnail, - ucid: ucid, - description: description_txt, - description_html: description_html, - video_count: video_count, - views: views, - updated: updated, - thumbnail: thumbnail, - subtitle: subtitle, - }) -end - -def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil) - # Show empty playlist if requested page is out of range - # (e.g, when a new playlist has been created, offset will be negative) - if offset >= playlist.video_count || offset < 0 - return [] of PlaylistVideo - end - - if playlist.is_a? InvidiousPlaylist - Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100) - else - if video_id - initial_data = YoutubeAPI.next({ - "videoId" => video_id, - "playlistId" => playlist.id, - }) - offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset - end - - videos = [] of PlaylistVideo | ProblematicTimelineItem - - until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count - # 100 videos per request - ctoken = produce_playlist_continuation(playlist.id, offset) - initial_data = YoutubeAPI.browse(ctoken) - videos += extract_playlist_videos(playlist.id, initial_data) - - offset += 100 - end - - return videos - end -end - -# TODO (2026-06-24): Migrate this function to use parsers instead, as it uses, -# the same LockupViewModel used in Channel videos and Youtube playlists that -# appears on searches (Invidious /search endpoint). -# Related to https://github.com/iv-org/invidious/pull/5736 -def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo | ProblematicTimelineItem - - if initial_data["contents"]? - tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] - tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"] - - # Watch out the two versions, with and without "s" - if tabs_renderer["contents"]? || tabs_renderer["content"]? - # Initial playlist data - tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"] - - list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0] - contents = list_renderer.["itemSectionRenderer"]["contents"].as_a - else - # Continuation data - contents = initial_data["onResponseReceivedActions"][0]? - .try &.["appendContinuationItemsAction"]["continuationItems"].as_a - end - else - contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a - end - - contents.try &.each do |item| - if i = item["lockupViewModel"]? - thumbnail_view_model = i.dig?( - "contentImage", "thumbnailViewModel" - ) - - watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") - video_id = watch_endpoint.try &.["videoId"]?.try &.as_s - plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id - index = watch_endpoint.try &.["index"]?.try &.as_i64 - - metadata = i["metadata"]? - lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel") - title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s - lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata") - metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a - - # Find the metadataParts with commandRuns inside, which contains author - # information. - metadata_parts = metadata_rows.try &.find { |row| - parts = row["metadataParts"]?.try &.as_a - parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a } - }.try &.["metadataParts"].as_a - - if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns")) - .try &.["text"] - author = author_info["content"].as_s - ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId") - .try &.as_s - end - - length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s - length_seconds = decode_length_seconds(length) if length - - live = false - - if !length_seconds - live = true - length_seconds = 0 - end - - videos << PlaylistVideo.new({ - title: title || "", - id: video_id || "", - author: author || "", - ucid: ucid || "", - length_seconds: length_seconds, - published: Time.utc, - plid: plid, - live_now: live, - index: index || -1_i64, - }) - end - rescue ex - videos << ProblematicTimelineItem.new(parse_exception: ex) - end - - return videos -end - -def template_playlist(playlist, listen) - html = <<-END_HTML -

- - #{playlist["title"]} - -

-
-
    - END_HTML - - playlist["videos"].as_a.each do |video| - html += <<-END_HTML -
  1. - -
    - -

    #{recode_length_seconds(video["lengthSeconds"].as_i)}

    -
    -

    #{video["title"]}

    -

    - #{video["author"]} -

    -     -
    2. - END_HTML - end - - html += <<-END_HTML -
-
-
- END_HTML - - html -end diff --git a/src/invidious/mixes.cr b/src/invidious/playlists/mixes.cr similarity index 97% rename from src/invidious/mixes.cr rename to src/invidious/playlists/mixes.cr index 7dd8a8309..cb4a5bdf9 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/playlists/mixes.cr @@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_" response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) - initial_data = Helpers.extract_initial_data(response.body) + initial_data = Invidious::Helpers.extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? raise InfoException.new("Could not create mix.") diff --git a/src/invidious/playlists/playlists.cr b/src/invidious/playlists/playlists.cr new file mode 100644 index 000000000..1091972a2 --- /dev/null +++ b/src/invidious/playlists/playlists.cr @@ -0,0 +1,583 @@ +struct PlaylistVideo + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property length_seconds : Int32 + property published : Time + property plid : String + property index : Int64 + property live_now : Bool + + def to_xml(xml : XML::Builder) + xml.element("entry") do + xml.element("id") { xml.text "yt:video:#{self.id}" } + xml.element("yt:videoId") { xml.text self.id } + xml.element("yt:channelId") { xml.text self.ucid } + xml.element("title") { xml.text self.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") + + xml.element("author") do + xml.element("name") { xml.text self.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } + end + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") + end + end + end + + xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } + + xml.element("media:group") do + xml.element("media:title") { xml.text self.title } + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", + width: "320", height: "180") + end + end + end + + def to_xml(_xml : Nil = nil) + XML.build { |xml| to_xml(xml) } + end + + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + + def to_json(json : JSON::Builder, index : Int32? = nil) + json.object do + json.field "type", "video" + + json.field "title", self.title + json.field "videoId", self.id + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", "/channel/#{self.ucid}" + + json.field "videoThumbnails" do + Invidious::JSONify::APIv1.thumbnails(json, self.id) + end + + if index + json.field "index", index + json.field "indexId", self.index.to_u64.to_s(16).upcase + else + json.field "index", self.index + end + + json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now + end + end + + def to_json(_json : Nil, index : Int32? = nil) + JSON.build { |json| to_json(json, index: index) } + end +end + +struct Playlist + include DB::Serializable + + property title : String + property id : String + property author : String + property author_thumbnail : String + property ucid : String + property description : String + property description_html : String + property video_count : Int32 + property views : Int64 + property updated : Time + property thumbnail : String? + property subtitle : String? + + def to_json(offset, json : JSON::Builder, video_id : String? = nil) + json.object do + json.field "type", "playlist" + json.field "title", self.title + json.field "playlistId", self.id + json.field "playlistThumbnail", self.thumbnail + + json.field "author", self.author + json.field "authorId", self.ucid + if !self.ucid.empty? + json.field "authorUrl", "/channel/#{self.ucid}" + else + json.field "authorUrl", "" + end + json.field "subtitle", self.subtitle + + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} + + qualities.each do |quality| + json.object do + json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end + end + end + end + + json.field "description", self.description + json.field "descriptionHtml", self.description_html + json.field "videoCount", self.video_count + + json.field "viewCount", self.views + json.field "updated", self.updated.to_unix + json.field "isListed", self.privacy.public? + + json.field "videos" do + json.array do + videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id) + videos.each do |video| + video.to_json(json) + end + end + end + end + end + + def to_json(offset, _json : Nil = nil, video_id : String? = nil) + JSON.build do |json| + to_json(offset, json, video_id: video_id) + end + end + + def privacy + PlaylistPrivacy::Public + end +end + +enum PlaylistPrivacy + Public = 0 + Unlisted = 1 + Private = 2 +end + +struct InvidiousPlaylist + include DB::Serializable + + property title : String + property id : String + property author : String + property description : String = "" + property video_count : Int32 + property created : Time + property updated : Time + + @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)] + property privacy : PlaylistPrivacy = PlaylistPrivacy::Private + property index : Array(Int64) + + @[DB::Field(ignore: true)] + property thumbnail_id : String? + + module PlaylistPrivacyConverter + def self.from_rs(rs) + return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) + end + end + + def to_json(offset, json : JSON::Builder, video_id : String? = nil) + json.object do + json.field "type", "invidiousPlaylist" + json.field "title", self.title + json.field "playlistId", self.id + + json.field "author", self.author + json.field "authorId", self.ucid + json.field "authorUrl", nil + json.field "authorThumbnails", [] of String + + json.field "description", Invidious::Helpers.html_to_content(self.description_html) + json.field "descriptionHtml", self.description_html + json.field "videoCount", self.video_count + + json.field "viewCount", self.views + json.field "updated", self.updated.to_unix + json.field "isListed", self.privacy.public? + + json.field "videos" do + json.array do + if (!offset || offset == 0) && !video_id.nil? + index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id) + offset = self.index.index(index) || 0 + end + + videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id) + videos.each_with_index do |video, idx| + video.to_json(json, offset + idx) + end + end + end + end + end + + def to_json(offset, _json : Nil = nil, video_id : String? = nil) + JSON.build do |json| + to_json(offset, json, video_id: video_id) + end + end + + def thumbnail + # TODO: Get playlist thumbnail from playlist data rather than first video + @thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------" + "/vi/#{@thumbnail_id}/mqdefault.jpg" + end + + def author_thumbnail + nil + end + + def ucid + nil + end + + def views + 0_i64 + end + + def description_html + HTML.escape(self.description) + end +end + +module Invidious::Playlists::Playlists + extend self + + def create_playlist(title, privacy, user) + plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" + + playlist = InvidiousPlaylist.new({ + title: title.byte_slice(0, 150), + id: plid, + author: user.email, + description: "", # Max 5000 characters + video_count: 0, + created: Time.utc, + updated: Time.utc, + privacy: privacy, + index: [] of Int64, + }) + + Invidious::Database::Playlists.insert(playlist) + + return playlist + end + + def subscribe_playlist(user, playlist) + playlist = InvidiousPlaylist.new({ + title: playlist.title[..150], + id: playlist.id, + author: user.email, + description: "", # Max 5000 characters + video_count: playlist.video_count, + created: Time.utc, + updated: playlist.updated, + privacy: PlaylistPrivacy::Private, + index: [] of Int64, + }) + + Invidious::Database::Playlists.insert(playlist) + + return playlist + end + + def produce_playlist_continuation(id, index) + if id.starts_with? "UC" + id = "UU" + id.lchop("UC") + end + plid = "VL" + id + + # Emulate a "request counter" increment, to make perfectly valid + # ctokens, even if at the time of writing, it's ignored by youtube. + request_count = (index / 100).to_i64 || 1_i64 + + data = {"1:varint" => index.to_i64} + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i, padding: false) } + + object = { + "80226972:embedded" => { + "2:string" => plid, + "3:base64" => { + "1:varint" => request_count, + "15:string" => "PT:#{data}", + "104:embedded" => {"1:0:varint" => 0_i64}, + }, + "35:string" => id, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end + + def get_playlist(plid : String) + if plid.starts_with? "IV" + if playlist = Invidious::Database::Playlists.select(id: plid) + return playlist + else + raise NotFoundException.new("Playlist does not exist.") + end + else + return fetch_playlist(plid) + end + end + + def fetch_playlist(plid : String) + if plid.starts_with? "UC" + plid = "UU#{plid.lchop("UC")}" + end + + initial_data = YoutubeAPI.browse("VL" + plid, params: "") + + playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items") + raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer + + playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer") + raise InfoException.new("Could not extract playlist info") if !playlist_info + + title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || "" + + desc_item = playlist_info["description"]? + + description_txt = desc_item.try &.["runs"]?.try &.as_a + .map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" + + description_html = desc_item.try &.["runs"]?.try &.as_a + .try { |run| content_to_comment_html(run).try &.to_s } || "

" + + thumbnail = playlist_info.dig?( + "thumbnailRenderer", "playlistVideoThumbnailRenderer", + "thumbnail", "thumbnails", 0, "url" + ).try &.as_s || playlist_info.dig?( + "thumbnailRenderer", "playlistCustomThumbnailRenderer", + "thumbnail", "thumbnails", 0, "url" + ).try &.as_s + + views = 0_i64 + updated = Time.utc + video_count = 0 + + subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle")) + + playlist_info["stats"]?.try &.as_a.each do |stat| + text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s + next if !text + + if text.includes? "video" + video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "view" + views = text.gsub(/\D/, "").to_i64? || 0_i64 + elsif !text.includes? "Pay to watch" + updated = decode_date(text.lchop("Last updated on ").lchop("Updated ")) + end + end + + if playlist_sidebar_renderer.size < 2 + author = "" + author_thumbnail = "" + ucid = "" + else + author_info = playlist_sidebar_renderer[1].dig?( + "playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer" + ) + + raise InfoException.new("Could not extract author info") if !author_info + + author = author_info.dig?("title", "runs", 0, "text").try &.as_s || "" + author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || "" + ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" + end + + return Playlist.new({ + title: title, + id: plid, + author: author, + author_thumbnail: author_thumbnail, + ucid: ucid, + description: description_txt, + description_html: description_html, + video_count: video_count, + views: views, + updated: updated, + thumbnail: thumbnail, + subtitle: subtitle, + }) + end + + def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil) + # Show empty playlist if requested page is out of range + # (e.g, when a new playlist has been created, offset will be negative) + if offset >= playlist.video_count || offset < 0 + return [] of PlaylistVideo + end + + if playlist.is_a? InvidiousPlaylist + Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100) + else + if video_id + initial_data = YoutubeAPI.next({ + "videoId" => video_id, + "playlistId" => playlist.id, + }) + offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset + end + + videos = [] of PlaylistVideo | ProblematicTimelineItem + + until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count + # 100 videos per request + ctoken = produce_playlist_continuation(playlist.id, offset) + initial_data = YoutubeAPI.browse(ctoken) + videos += extract_playlist_videos(playlist.id, initial_data) + + offset += 100 + end + + return videos + end + end + + # TODO (2026-06-24): Migrate this function to use parsers instead, as it uses, + # the same LockupViewModel used in Channel videos and Youtube playlists that + # appears on searches (Invidious /search endpoint). + # Related to https://github.com/iv-org/invidious/pull/5736 + def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any)) + videos = [] of PlaylistVideo | ProblematicTimelineItem + + if initial_data["contents"]? + tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] + tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"] + + # Watch out the two versions, with and without "s" + if tabs_renderer["contents"]? || tabs_renderer["content"]? + # Initial playlist data + tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"] + + list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0] + contents = list_renderer.["itemSectionRenderer"]["contents"].as_a + else + # Continuation data + contents = initial_data["onResponseReceivedActions"][0]? + .try &.["appendContinuationItemsAction"]["continuationItems"].as_a + end + else + contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a + end + + contents.try &.each do |item| + if i = item["lockupViewModel"]? + thumbnail_view_model = i.dig?( + "contentImage", "thumbnailViewModel" + ) + + watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + video_id = watch_endpoint.try &.["videoId"]?.try &.as_s + plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id + index = watch_endpoint.try &.["index"]?.try &.as_i64 + + metadata = i["metadata"]? + lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel") + title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s + lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata") + metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a + + # Find the metadataParts with commandRuns inside, which contains author + # information. + metadata_parts = metadata_rows.try &.find { |row| + parts = row["metadataParts"]?.try &.as_a + parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a } + }.try &.["metadataParts"].as_a + + if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns")) + .try &.["text"] + author = author_info["content"].as_s + ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId") + .try &.as_s + end + + length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s + length_seconds = decode_length_seconds(length) if length + + live = false + + if !length_seconds + live = true + length_seconds = 0 + end + + videos << PlaylistVideo.new({ + title: title || "", + id: video_id || "", + author: author || "", + ucid: ucid || "", + length_seconds: length_seconds, + published: Time.utc, + plid: plid, + live_now: live, + index: index || -1_i64, + }) + end + rescue ex + videos << ProblematicTimelineItem.new(parse_exception: ex) + end + + return videos + end + + def template_playlist(playlist, listen) + html = <<-END_HTML +

+ + #{playlist["title"]} + +

+
+
    + END_HTML + + playlist["videos"].as_a.each do |video| + html += <<-END_HTML +
  1. + +
    + +

    #{recode_length_seconds(video["lengthSeconds"].as_i)}

    +
    +

    #{video["title"]}

    +

    + #{video["author"]} +

    +     +
    2. + END_HTML + end + + html += <<-END_HTML +
+
+
+ END_HTML + + html + end +end diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index c8db207c2..5028fc8f5 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -21,7 +21,7 @@ module Invidious::Routes::Account user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":change_password"}, HMAC_KEY) templated "user/change_password" end @@ -43,33 +43,33 @@ module Invidious::Routes::Account token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end password = env.params.body["password"]? if password.nil? || password.empty? - return error_template(401, "Password is a required field") + return Errors.error_template(401, "Password is a required field") end new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 - return error_template(400, "New passwords must match") + return Errors.error_template(400, "New passwords must match") end new_password = new_passwords.uniq[0] if new_password.empty? - return error_template(401, "Password cannot be empty") + return Errors.error_template(401, "Password cannot be empty") end if new_password.bytesize > 55 - return error_template(400, "Password cannot be longer than 55 characters") + return Errors.error_template(400, "Password cannot be longer than 55 characters") end if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - return error_template(401, "Incorrect password") + return Errors.error_template(401, "Incorrect password") end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) @@ -96,7 +96,7 @@ module Invidious::Routes::Account user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_account"}, HMAC_KEY) templated "user/delete_account" end @@ -118,9 +118,9 @@ module Invidious::Routes::Account token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end view_name = "subscriptions_#{sha256(user.email)}" @@ -154,7 +154,7 @@ module Invidious::Routes::Account user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":clear_watch_history"}, HMAC_KEY) templated "user/clear_watch_history" end @@ -176,9 +176,9 @@ module Invidious::Routes::Account token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end Invidious::Database::Users.clear_watch_history(user) @@ -203,7 +203,7 @@ module Invidious::Routes::Account user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY) scopes = env.params.query["scopes"]?.try &.split(",") scopes ||= [] of String @@ -235,16 +235,16 @@ module Invidious::Routes::Account token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? - access_token = generate_token(user.email, scopes, expire, HMAC_KEY) + access_token = Invidious::Helpers::Tokens.generate_token(user.email, scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -310,7 +310,7 @@ module Invidious::Routes::Account if redirect return env.redirect referer else - return error_json(403, "No such user") + return Errors.error_json(403, "No such user") end end @@ -319,12 +319,12 @@ module Invidious::Routes::Account token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect - return error_template(400, ex) + return Errors.error_template(400, ex) else - return error_json(400, ex) + return Errors.error_json(400, ex) end end @@ -333,7 +333,7 @@ module Invidious::Routes::Account session = env.params.query["session"] Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else - return error_json(400, "Unsupported action #{action}") + return Errors.error_json(400, "Unsupported action #{action}") end if redirect diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index da706a9ef..c9497d462 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Authenticated # topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) # topics ||= [] of String - # Helpers.create_notification_stream(env, topics, connection_channel) + # Invidious::Helpers.create_notification_stream(env, topics, connection_channel) # end def self.get_preferences(env) @@ -78,13 +78,13 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) if !user.preferences.watch_history - return error_json(409, "Watch history is disabled in preferences.") + return Errors.error_json(409, "Watch history is disabled in preferences.") end # Sanity checks id = env.params.url["id"] unless validate_video_id(id) - return error_json(400, InvalidVideoID.new(id)) + return Errors.error_json(400, InvalidVideoID.new(id)) end Invidious::Database::Users.mark_watched(user, id) @@ -95,12 +95,12 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) if !user.preferences.watch_history - return error_json(409, "Watch history is disabled in preferences.") + return Errors.error_json(409, "Watch history is disabled in preferences.") end video_id = env.params.url["id"] unless video_id && validate_video_id(video_id) - return error_json(400, InvalidVideoID.new(video_id)) + return Errors.error_json(400, InvalidVideoID.new(video_id)) end Invidious::Database::Users.mark_unwatched(user, video_id) @@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(user, max_results, page) + videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page) JSON.build do |json| json.object do @@ -175,7 +175,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid) + Invidious::Channels::Channels.get_channel(ucid) Invidious::Database::Users.subscribe_channel(user, ucid) end @@ -214,19 +214,19 @@ module Invidious::Routes::API::V1::Authenticated title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) if !title - return error_json(400, "Invalid title.") + return Errors.error_json(400, "Invalid title.") end privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } if !privacy - return error_json(400, "Invalid privacy setting.") + return Errors.error_json(400, "Invalid privacy setting.") end if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 - return error_json(400, "User cannot have more than 100 playlists.") + return Errors.error_json(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(title, privacy, user) + playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.status_code = 201 { @@ -241,16 +241,16 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"]? if !plid || plid.empty? - return error_json(400, "A playlist ID is required") + return Errors.error_json(400, "A playlist ID is required") end playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end if playlist.author != user.email - return error_json(403, "Invalid user") + return Errors.error_json(403, "Invalid user") end title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title @@ -278,11 +278,11 @@ module Invidious::Routes::API::V1::Authenticated playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end if playlist.author != user.email - return error_json(403, "Invalid user") + return Errors.error_json(403, "Invalid user") end Invidious::Database::Playlists.delete(plid) @@ -298,29 +298,29 @@ module Invidious::Routes::API::V1::Authenticated playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end if playlist.author != user.email - return error_json(403, "Invalid user") + return Errors.error_json(403, "Invalid user") end if playlist.index.size >= CONFIG.playlist_length_limit - return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") + return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end video_id = env.params.json["videoId"].try &.as(String) # Sanity checks unless video_id && validate_video_id(video_id) - return error_json(400, InvalidVideoID.new(video_id)) + return Errors.error_json(400, InvalidVideoID.new(video_id)) end begin video = get_video(video_id) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end playlist_video = PlaylistVideo.new({ @@ -355,15 +355,15 @@ module Invidious::Routes::API::V1::Authenticated playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end if playlist.author != user.email - return error_json(403, "Invalid user") + return Errors.error_json(403, "Invalid user") end if !playlist.index.includes? index - return error_json(404, "Playlist does not contain index") + return Errors.error_json(404, "Playlist does not contain index") end Invidious::Database::PlaylistVideos.delete(index, plid) @@ -411,7 +411,7 @@ module Invidious::Routes::API::V1::Authenticated callback_url = env.params.json["callbackUrl"]?.try &.as(String) expire = env.params.json["expire"]?.try &.as(Int64) else - return error_json(400, "Invalid or missing header 'Content-Type'") + return Errors.error_json(400, "Invalid or missing header 'Content-Type'") end if callback_url && callback_url.empty? @@ -427,7 +427,7 @@ module Invidious::Routes::API::V1::Authenticated # Used by template bellow. # ameba:disable Lint/UselessAssign - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) return templated "user/authorize_token" else env.response.content_type = "application/json" @@ -436,12 +436,12 @@ module Invidious::Routes::API::V1::Authenticated authorized_scopes = [] of String scopes.each do |scope| - if scopes_include_scope(superset_scopes, scope) + if Invidious::Helpers::Tokens.scopes_include_scope(superset_scopes, scope) authorized_scopes << scope end end - access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY) + access_token = Invidious::Helpers::Tokens.generate_token(user.email, authorized_scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -473,10 +473,10 @@ module Invidious::Routes::API::V1::Authenticated # Allow tokens to revoke other tokens with correct scope if session == env.get("session").as(String) Invidious::Database::SessionIDs.delete(sid: session) - elsif scopes_include_scope(scopes, "GET:tokens") + elsif Invidious::Helpers::Tokens.scopes_include_scope(scopes, "GET:tokens") Invidious::Database::SessionIDs.delete(sid: session) else - return error_json(400, "Cannot revoke session #{session}") + return Errors.error_json(400, "Cannot revoke session #{session}") end env.response.status_code = 204 @@ -489,6 +489,6 @@ module Invidious::Routes::API::V1::Authenticated topics = raw_topics.try &.split(",").uniq!.first(1000) topics ||= [] of String - Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL) + Invidious::Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL) end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 8c21fe19c..00512cb5a 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -3,14 +3,14 @@ module Invidious::Routes::API::V1::Channels # This sets the `channel` variable, or handles Exceptions. private macro get_channel begin - channel = get_about_info(ucid) + channel = Invidious::Channels::About.get_about_info(ucid) rescue ex : ChannelRedirect env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) + return Errors.error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -29,8 +29,8 @@ module Invidious::Routes::API::V1::Channels if channel.is_age_gated begin - playlist = get_playlist(channel.ucid.sub("UC", "UULF")) - videos = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF")) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. videos = [] of PlaylistVideo @@ -39,7 +39,7 @@ module Invidious::Routes::API::V1::Channels begin videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Channels json.field "autoGenerated", channel.auto_generated json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly - json.field "description", Helpers.html_to_content(channel.description_html) + json.field "description", Invidious::Helpers.html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html json.field "allowedRegions", channel.allowed_regions @@ -117,7 +117,7 @@ module Invidious::Routes::API::V1::Channels json.array do # Fetch related channels begin - related_channels, _ = fetch_related_channels(channel) + related_channels, _ = Invidious::Channels::About.fetch_related_channels(channel) rescue ex related_channels = [] of SearchChannel end @@ -155,8 +155,8 @@ module Invidious::Routes::API::V1::Channels if channel.is_age_gated begin - playlist = get_playlist(channel.ucid.sub("UC", "UULF")) - videos = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF")) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. videos = [] of PlaylistVideo @@ -168,7 +168,7 @@ module Invidious::Routes::API::V1::Channels channel, continuation: continuation, sort_by: sort_by ) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -201,8 +201,8 @@ module Invidious::Routes::API::V1::Channels if channel.is_age_gated begin - playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) - videos = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. videos = [] of PlaylistVideo @@ -214,7 +214,7 @@ module Invidious::Routes::API::V1::Channels channel, continuation: continuation, sort_by: sort_by ) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -247,8 +247,8 @@ module Invidious::Routes::API::V1::Channels if channel.is_age_gated begin - playlist = get_playlist(channel.ucid.sub("UC", "UULV")) - videos = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV")) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. videos = [] of PlaylistVideo @@ -260,7 +260,7 @@ module Invidious::Routes::API::V1::Channels channel, continuation: continuation, sort_by: sort_by ) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -292,7 +292,7 @@ module Invidious::Routes::API::V1::Channels channel = nil # Make the compiler happy get_channel() - items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) JSON.build do |json| json.object do @@ -321,7 +321,7 @@ module Invidious::Routes::API::V1::Channels channel = nil # Make the compiler happy get_channel() - items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation) + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts(channel.ucid, channel.author, continuation) JSON.build do |json| json.object do @@ -350,7 +350,7 @@ module Invidious::Routes::API::V1::Channels channel = nil # Make the compiler happy get_channel() - items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation) + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases(channel.ucid, channel.author, continuation) JSON.build do |json| json.object do @@ -379,7 +379,7 @@ module Invidious::Routes::API::V1::Channels channel = nil # Make the compiler happy get_channel() - items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation) + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses(channel.ucid, channel.author, continuation) JSON.build do |json| json.object do @@ -413,9 +413,9 @@ module Invidious::Routes::API::V1::Channels # sort_by = env.params.query["sort_by"]?.try &.downcase begin - fetch_channel_community(ucid, continuation, locale, format, thin_mode) + Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, format, thin_mode) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -434,16 +434,16 @@ module Invidious::Routes::API::V1::Channels if ucid.nil? response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") - return error_json(400, "Invalid post ID") if response["error"]? - ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) + return Errors.error_json(400, "Invalid post ID") if response["error"]? + ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) else ucid = ucid.to_s end begin - fetch_channel_community_post(ucid, id, locale, format, thin_mode) + Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, format, thin_mode) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -487,9 +487,9 @@ module Invidious::Routes::API::V1::Channels continuation = env.params.query["continuation"]? begin - items, next_continuation = fetch_related_channels(channel, continuation) + items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end JSON.build do |json| @@ -520,7 +520,7 @@ module Invidious::Routes::API::V1::Channels begin search_results = query.process rescue ex - return error_json(400, ex) + return Errors.error_json(400, ex) end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 46618008a..cbde8abd4 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -8,9 +8,9 @@ module Invidious::Routes::API::V1::Feeds trending_type = env.params.query["type"]? begin - trending, _ = fetch_trending(trending_type, region, locale) + trending, _ = Invidious::Feeds::Trending.fetch(trending_type, region, locale) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end videos = JSON.build do |json| diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 4ae877a8b..2a3119435 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -50,16 +50,16 @@ module Invidious::Routes::API::V1::Misc end begin - playlist = get_playlist(plid) + playlist = Invidious::Playlists::Playlists.get_playlist(plid) rescue ex : InfoException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end user = env.get?("user").try &.as(User) if !playlist || playlist.privacy.private? && playlist.author != user.try &.email - return error_json(404, "Playlist does not exist.") + return Errors.error_json(404, "Playlist does not exist.") end # includes into the playlist a maximum of 20 videos, before the offset @@ -88,7 +88,7 @@ module Invidious::Routes::API::V1::Misc end if format == "html" - playlist_html = template_playlist(json_response, listen) + playlist_html = Invidious::Playlists::Playlists.template_playlist(json_response, listen) index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} response = { @@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Misc mix.videos = mix.videos[index..-1] rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end response = JSON.build do |json| @@ -178,14 +178,14 @@ module Invidious::Routes::API::V1::Misc env.response.content_type = "application/json" url = env.params.query["url"]? - return error_json(400, "Missing URL to resolve") if !url + return Errors.error_json(400, "Missing URL to resolve") if !url begin resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" if page_type == "WEB_PAGE_TYPE_UNKNOWN" - return error_json(400, "Unknown url") + return Errors.error_json(400, "Unknown url") end sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint @@ -204,7 +204,7 @@ module Invidious::Routes::API::V1::Misc post_id = nil end rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end JSON.build do |json| json.object do diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 59a30745e..14b19c837 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -10,7 +10,7 @@ module Invidious::Routes::API::V1::Search begin search_results = query.process rescue ex - return error_json(400, ex) + return Errors.error_json(400, ex) end JSON.build do |json| @@ -53,7 +53,7 @@ module Invidious::Routes::API::V1::Search end end rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -67,9 +67,9 @@ module Invidious::Routes::API::V1::Search env.response.content_type = "application/json" begin - results = Invidious::Hashtag.fetch(hashtag, page, region) + results = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page, region) rescue ex - return error_json(400, ex) + return Errors.error_json(400, ex) end JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 14221ed03..da55e39e1 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -16,9 +16,9 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id, region: region) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end return JSON.build do |json| @@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Videos # Sanity checks unless validate_video_id(id) - return error_json(400, InvalidVideoID.new(id)) + return Errors.error_json(400, InvalidVideoID.new(id)) end # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 @@ -301,7 +301,7 @@ module Invidious::Routes::API::V1::Videos annotations = response.body - Helpers.cache_annotation(video_id, annotations) + Invidious::Helpers.cache_annotation(video_id, annotations) end else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{video_id}") @@ -354,9 +354,9 @@ module Invidious::Routes::API::V1::Videos begin comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end return comments @@ -371,7 +371,7 @@ module Invidious::Routes::API::V1::Videos end if !reddit_thread || !comments - return error_json(404, "No reddit threads found") + return Errors.error_json(404, "No reddit threads found") end if format == "json" @@ -404,10 +404,10 @@ module Invidious::Routes::API::V1::Videos proxy = {"1", "true"}.any? &.== env.params.query["local"]? response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") - return error_json(400, "Invalid clip ID") if response["error"]? + return Errors.error_json(400, "Invalid clip ID") if response["error"]? video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s - return error_json(400, "Invalid clip ID") if video_id.nil? + return Errors.error_json(400, "Invalid clip ID") if video_id.nil? start_time = nil end_time = nil @@ -420,9 +420,9 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(video_id, region: region) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end return JSON.build do |json| @@ -454,9 +454,9 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end response = JSON.build do |json| @@ -494,14 +494,14 @@ module Invidious::Routes::API::V1::Videos begin video = get_video(id) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end target_transcript = video.captions.select(&.name.== label) if target_transcript.empty? - return error_json(404, NotFoundException.new("Requested transcript does not exist")) + return Errors.error_json(404, NotFoundException.new("Requested transcript does not exist")) else target_transcript = target_transcript[0] lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated @@ -515,9 +515,9 @@ module Invidious::Routes::API::V1::Videos YoutubeAPI.get_transcript(params), lang, auto_generated ) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex - return error_json(500, ex) + return Errors.error_json(500, ex) end return transcript.to_json diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index b75574b69..f8a4869e0 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -1,4 +1,6 @@ module Invidious::Routes::BeforeAll + extend self + struct CompanionCSP property companion_urls : String = "" @@ -89,7 +91,7 @@ module Invidious::Routes::BeforeAll if email = Database::SessionIDs.select_email(sid) user = Database::Users.select!(email: email) - csrf_token = generate_response(sid, { + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, { ":authorize_token", ":playlist_ajax", ":signout", @@ -107,7 +109,7 @@ module Invidious::Routes::BeforeAll end end - dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s + dark_mode = Invidious::User::Converters.convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s thin_mode = env.params.query["thin_mode"]? thin_mode = (thin_mode == "true") || preferences.thin_mode locale = env.params.query["hl"]? || preferences.locale diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index ff8b0f270..a77a66aed 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -23,7 +23,7 @@ module Invidious::Routes::Channels sort_by ||= "last" sort_options = {"last", "oldest", "newest"} - items, next_continuation = fetch_channel_playlists( + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists( channel.ucid, channel.author, continuation, sort_by ) @@ -42,8 +42,8 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String begin - playlist = get_playlist(channel.ucid.sub("UC", "UULF")) - items = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF")) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. items = [] of PlaylistVideo @@ -77,8 +77,8 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String begin - playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) - items = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH")) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. items = [] of PlaylistVideo @@ -112,8 +112,8 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String begin - playlist = get_playlist(channel.ucid.sub("UC", "UULV")) - items = get_playlist_videos(playlist, offset: 0) + playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV")) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) rescue ex : InfoException # playlist doesnt exist. items = [] of PlaylistVideo @@ -146,7 +146,7 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - items, next_continuation = fetch_channel_playlists( + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists( channel.ucid, channel.author, continuation, (sort_by || "last") ) @@ -166,7 +166,7 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String - items, next_continuation = fetch_channel_podcasts( + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts( channel.ucid, channel.author, continuation ) @@ -186,7 +186,7 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String - items, next_continuation = fetch_channel_releases( + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases( channel.ucid, channel.author, continuation ) @@ -206,7 +206,7 @@ module Invidious::Routes::Channels sort_by = "" sort_options = [] of String - items, next_continuation = fetch_channel_courses( + items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses( channel.ucid, channel.author, continuation ) @@ -247,7 +247,7 @@ module Invidious::Routes::Channels sort_options = [] of String begin - items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) + items = JSON.parse(Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) rescue ex : InfoException env.response.status_code = 500 error_message = ex.message @@ -255,7 +255,7 @@ module Invidious::Routes::Channels env.response.status_code = 404 error_message = ex.message rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end templated "community" @@ -280,14 +280,14 @@ module Invidious::Routes::Channels if !ucid.nil? ucid = ucid.to_s - post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode) else # resolve the url to get the author's UCID response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") - return error_template(400, "Invalid post ID") if response["error"]? + return Errors.error_template(400, "Invalid post ID") if response["error"]? - ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) - post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) + ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) + post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode) end post_response = JSON.parse(post_response) @@ -309,7 +309,7 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - items, next_continuation = fetch_related_channels(channel, continuation) + items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation) # Featured/related channels can't be sorted sort_options = [] of String @@ -354,7 +354,7 @@ module Invidious::Routes::Channels resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"] rescue ex : InfoException | KeyError - return error_template(404, I18n.translate(locale, "This channel does not exist.")) + return Errors.error_template(404, I18n.translate(locale, "This channel does not exist.")) end selected_tab = env.params.url["tab"]? @@ -380,7 +380,7 @@ module Invidious::Routes::Channels user = env.params.query["user"]? if !user - return error_template(404, "This channel does not exist.") + return Errors.error_template(404, "This channel does not exist.") else env.redirect "/user/#{user}#{uri_params}" end @@ -433,13 +433,13 @@ module Invidious::Routes::Channels continuation = env.params.query["continuation"]? begin - channel = get_about_info(ucid) + channel = Invidious::Channels::About.get_about_info(ucid) rescue ex : ChannelRedirect return env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end env.set "search", "channel:#{ucid} " diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 41dc517c2..d12fc7140 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -5,9 +5,9 @@ module Invidious::Routes::Embed locale = env.get("preferences").as(Preferences).locale if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin - playlist = get_playlist(plid) + playlist = Invidious::Playlists::Playlists.get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(playlist, offset: offset) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset) if videos.empty? url = "/playlist?list=#{plid}" raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url)) @@ -15,9 +15,9 @@ module Invidious::Routes::Embed first_playlist_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end url = "/embed/#{first_playlist_video.id}?#{env.params.query}" @@ -66,9 +66,9 @@ module Invidious::Routes::Embed if plid begin - playlist = get_playlist(plid) + playlist = Invidious::Playlists::Playlists.get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(playlist, offset: offset) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset) if videos.empty? url = "/playlist?list=#{plid}" raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url)) @@ -76,9 +76,9 @@ module Invidious::Routes::Embed first_playlist_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end url = "/embed/#{first_playlist_video.id}" @@ -101,7 +101,7 @@ module Invidious::Routes::Embed env.params.query.delete_all("channel") if !video_id || video_id == "live_stream" - return error_template(500, "Video is unavailable.") + return Errors.error_template(500, "Video is unavailable.") end url = "/embed/#{video_id}" @@ -135,9 +135,9 @@ module Invidious::Routes::Embed begin video = get_video(id, region: params.region) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end if preferences.annotations_subscribed && diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index ebb5f04b1..5fa32a973 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -53,9 +53,9 @@ module Invidious::Routes::Feeds region ||= preferences.region begin - trending, plid = fetch_trending(trending_type, region, locale) + trending, plid = Invidious::Feeds::Trending.fetch(trending_type, region, locale) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end templated "feeds/trending" @@ -91,7 +91,7 @@ module Invidious::Routes::Feeds page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(user, max_results, page) + videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page) if CONFIG.enable_user_notifications # "updated" here is used for delivering new notifications, so if @@ -150,7 +150,7 @@ module Invidious::Routes::Feeds if env.params.url["ucid"].matches?(/^[\w-]+$/) ucid = env.params.url["ucid"] else - return error_atom(400, InfoException.new("Invalid channel ucid provided.")) + return Errors.error_atom(400, InfoException.new("Invalid channel ucid provided.")) end params = HTTP::Params.parse(env.params.query["params"]? || "") @@ -162,7 +162,7 @@ module Invidious::Routes::Feeds } response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}") - return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404 + return Errors.error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404 rss = XML.parse(response.body) videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry| @@ -250,7 +250,7 @@ module Invidious::Routes::Feeds params = HTTP::Params.parse(env.params.query["params"]? || "") - videos, notifications = get_subscription_feed(user, max_results, page) + videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page) XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -281,11 +281,11 @@ module Invidious::Routes::Feeds if plid.starts_with? "IV" if playlist = Invidious::Database::Playlists.select(id: plid) - videos = get_playlist_videos(playlist, offset: 0) + videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0) user = env.get?("user").try &.as(User) if !playlist || playlist.privacy.private? && playlist.author != user.try &.email - return error_atom(404, "Playlist does not exist.") + return Errors.error_atom(404, "Playlist does not exist.") end return XML.build(indent: " ", encoding: "UTF-8") do |xml| @@ -317,7 +317,7 @@ module Invidious::Routes::Feeds end response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") - return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404 + return Errors.error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404 document = XML.parse(response.body) document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index d76bcc582..432bccd8d 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -106,7 +106,7 @@ module Invidious::Routes::Images break end - Helpers.proxy_file(response, env) + Invidious::Helpers.proxy_file(response, env) end rescue ex end @@ -158,6 +158,6 @@ module Invidious::Routes::Images return env.response.headers.delete("Transfer-Encoding") end - return Helpers.proxy_file(response, env) + return Invidious::Helpers.proxy_file(response, env) end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 7d7da4879..ea6e50da5 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -11,7 +11,7 @@ module Invidious::Routes::Login return env.redirect referer if user if !CONFIG.login_enabled - return error_template(400, "Login has been disabled by administrator.") + return Errors.error_template(400, "Login has been disabled by administrator.") end email = nil @@ -31,7 +31,7 @@ module Invidious::Routes::Login referer = get_referer(env, "/feed/subscriptions") if !CONFIG.login_enabled - return error_template(403, "Login has been disabled by administrator.") + return Errors.error_template(403, "Login has been disabled by administrator.") end # https://stackoverflow.com/a/574698 @@ -44,11 +44,11 @@ module Invidious::Routes::Login case account_type when "invidious" if email.nil? || email.empty? - return error_template(401, "User ID is a required field") + return Errors.error_template(401, "User ID is a required field") end if password.nil? || password.empty? - return error_template(401, "Password is a required field") + return Errors.error_template(401, "Password is a required field") end user = Invidious::Database::Users.select(email: email) @@ -64,7 +64,7 @@ module Invidious::Routes::Login env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) end else - return error_template(401, "Wrong username or password") + return Errors.error_template(401, "Wrong username or password") end # Since this user has already registered, we don't want to overwrite their preferences @@ -75,16 +75,16 @@ module Invidious::Routes::Login end else if !CONFIG.registration_enabled - return error_template(400, "Registration has been disabled by administrator.") + return Errors.error_template(400, "Registration has been disabled by administrator.") end if password.empty? - return error_template(401, "Password cannot be empty") + return Errors.error_template(401, "Password cannot be empty") end # See https://security.stackexchange.com/a/39851 if password.bytesize > 55 - return error_template(400, "Password cannot be longer than 55 characters") + return Errors.error_template(400, "Password cannot be longer than 55 characters") end password = password.byte_slice(0, 55) @@ -102,11 +102,11 @@ module Invidious::Routes::Login answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) rescue ex : InfoException - return error_template(400, InfoException.new("Erroneous CAPTCHA")) + return Errors.error_template(400, InfoException.new("Erroneous CAPTCHA")) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end else return templated "user/login" @@ -114,7 +114,7 @@ module Invidious::Routes::Login end sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user, sid = create_user(sid, email, password) + user, sid = Invidious::User::Users.create_user(sid, email, password) if language_header = env.request.headers["Accept-Language"]? if language = ANG.language_negotiator.best(language_header, I18n::LOCALES.keys) @@ -126,7 +126,7 @@ module Invidious::Routes::Login Invidious::Database::SessionIDs.insert(sid, email) view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(user.email)}") if alt = CONFIG.alternative_domains.index(host) env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid) @@ -166,9 +166,9 @@ module Invidious::Routes::Login token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end Invidious::Database::SessionIDs.delete(sid: sid) diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr index 8922b7405..b5e6b4ab5 100644 --- a/src/invidious/routes/notifications.cr +++ b/src/invidious/routes/notifications.cr @@ -18,7 +18,7 @@ module Invidious::Routes::Notifications if redirect return env.redirect referer else - return error_json(403, "No such user") + return Errors.error_json(403, "No such user") end end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 3a633d507..aec61f83d 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Playlists user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":create_playlist"}, HMAC_KEY) templated "create_playlist" end @@ -31,26 +31,26 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end title = env.params.body["title"]?.try &.as(String) if !title || title.empty? - return error_template(400, "Title cannot be empty.") + return Errors.error_template(400, "Title cannot be empty.") end privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") if !privacy - return error_template(400, "Invalid privacy setting.") + return Errors.error_template(400, "Invalid privacy setting.") end if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 - return error_template(400, "User cannot have more than 100 playlists.") + return Errors.error_template(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(title, privacy, user) + playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user) env.redirect "/playlist?list=#{playlist.id}" end @@ -67,13 +67,13 @@ module Invidious::Routes::Playlists playlist_id = env.params.query["list"] begin - playlist = get_playlist(playlist_id) + playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end - subscribe_playlist(user, playlist) + Invidious::Playlists::Playlists.subscribe_playlist(user, playlist) env.redirect "/playlist?list=#{playlist.id}" end @@ -92,7 +92,7 @@ module Invidious::Routes::Playlists plid = env.params.query["list"]? if !plid || plid.empty? - return error_template(400, "A playlist ID is required") + return Errors.error_template(400, "A playlist ID is required") end playlist = Invidious::Database::Playlists.select(id: plid) @@ -100,7 +100,7 @@ module Invidious::Routes::Playlists return env.redirect referer end - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_playlist"}, HMAC_KEY) templated "delete_playlist" end @@ -122,9 +122,9 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end playlist = Invidious::Database::Playlists.select(id: plid) @@ -163,12 +163,12 @@ module Invidious::Routes::Playlists end begin - items = get_playlist_videos(playlist, offset: (page - 1) * 100) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100) rescue ex items = [] of PlaylistVideo end - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY) + csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":edit_playlist"}, HMAC_KEY) # Pagination page_nav_html = Frontend::Pagination.nav_numeric(locale, @@ -197,9 +197,9 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex - return error_template(400, ex) + return Errors.error_template(400, ex) end playlist = Invidious::Database::Playlists.select(id: plid) @@ -286,7 +286,7 @@ module Invidious::Routes::Playlists if redirect return env.redirect referer else - return error_json(403, "No such user") + return Errors.error_json(403, "No such user") end end @@ -295,26 +295,26 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect - return error_template(400, ex) + return Errors.error_template(400, ex) else - return error_json(400, ex) + return Errors.error_json(400, ex) end end begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(playlist_id).as(InvidiousPlaylist) + playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id).as(InvidiousPlaylist) raise "Invalid user" if playlist.author != user.email rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex if redirect - return error_template(400, ex) + return Errors.error_template(400, ex) else - return error_json(400, ex) + return Errors.error_json(400, ex) end end @@ -322,9 +322,9 @@ module Invidious::Routes::Playlists when "add_video" if playlist.index.size >= CONFIG.playlist_length_limit if redirect - return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") + return Errors.error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") else - return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") + return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end end @@ -333,12 +333,12 @@ module Invidious::Routes::Playlists begin video = get_video(video_id) rescue ex : NotFoundException - return error_json(404, ex) + return Errors.error_json(404, ex) rescue ex if redirect - return error_template(500, ex) + return Errors.error_template(500, ex) else - return error_json(500, ex) + return Errors.error_json(500, ex) end end @@ -359,7 +359,7 @@ module Invidious::Routes::Playlists when "remove_video" index = env.params.query["set_video_id"].to_i64? if index.nil? || !playlist.index.includes? index - return error_json(404, "Playlist does not contain index") + return Errors.error_json(404, "Playlist does not contain index") end Invidious::Database::PlaylistVideos.delete(index, playlist_id) @@ -367,9 +367,9 @@ module Invidious::Routes::Playlists when "move_video_before" # TODO: Playlist stub when nil - return error_json(400, "Missing action") + return Errors.error_json(400, "Missing action") else - return error_json(400, "Unsupported action #{action}") + return Errors.error_json(400, "Unsupported action #{action}") end if redirect @@ -399,11 +399,11 @@ module Invidious::Routes::Playlists end begin - playlist = get_playlist(plid) + playlist = Invidious::Playlists::Playlists.get_playlist(plid) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end if playlist.is_a? InvidiousPlaylist @@ -419,17 +419,17 @@ module Invidious::Routes::Playlists end if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email - return error_template(403, "This playlist is private.") + return Errors.error_template(403, "This playlist is private.") end begin if playlist.is_a? InvidiousPlaylist - items = get_playlist_videos(playlist, offset: (page - 1) * 100) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100) else - items = get_playlist_videos(playlist, offset: (page - 1) * 200) + items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 200) end rescue ex - return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") + return Errors.error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") end if playlist.author == user.try &.email @@ -460,7 +460,7 @@ module Invidious::Routes::Playlists begin mix = fetch_mix(rdid, continuation, locale: locale) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end templated "mix" diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 77ea02e91..ef0c4f4e3 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -329,7 +329,7 @@ module Invidious::Routes::PreferencesRoute if !success haltf(env, status_code: 415, - response: error_template(415, "Invalid subscription file uploaded") + response: Errors.error_template(415, "Invalid subscription file uploaded") ) end when "import_youtube_pl" @@ -338,7 +338,7 @@ module Invidious::Routes::PreferencesRoute if !success haltf(env, status_code: 415, - response: error_template(415, "Invalid playlist file uploaded") + response: Errors.error_template(415, "Invalid playlist file uploaded") ) end when "import_youtube_wh" @@ -347,7 +347,7 @@ module Invidious::Routes::PreferencesRoute if !success haltf(env, status_code: 415, - response: error_template(415, "Invalid watch history file uploaded") + response: Errors.error_template(415, "Invalid watch history file uploaded") ) end when "import_freetube" @@ -359,7 +359,7 @@ module Invidious::Routes::PreferencesRoute if !success haltf(env, status_code: 415, - response: error_template(415, "Uploaded file is too large") + response: Errors.error_template(415, "Uploaded file is too large") ) end else nil # Ignore diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 968a5554d..022a348af 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -71,9 +71,9 @@ module Invidious::Routes::Search items = query.process end rescue ex : ChannelSearchException - return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") + return Errors.error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end redirect_url = Invidious::Frontend::Misc.redirect_url(env) @@ -100,7 +100,7 @@ module Invidious::Routes::Search hashtag = env.params.url["hashtag"]? if hashtag.nil? || hashtag.empty? - return error_template(400, "Invalid request") + return Errors.error_template(400, "Invalid request") end page = env.params.query["page"]? @@ -112,9 +112,9 @@ module Invidious::Routes::Search end begin - items = Invidious::Hashtag.fetch(hashtag, page) + items = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end # Pagination diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 1de655d20..6c7422116 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -14,7 +14,7 @@ module Invidious::Routes::Subscriptions if redirect return env.redirect referer else - return error_json(403, "No such user") + return Errors.error_json(403, "No such user") end end @@ -23,12 +23,12 @@ module Invidious::Routes::Subscriptions token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect - return error_template(400, ex) + return Errors.error_template(400, ex) else - return error_json(400, ex) + return Errors.error_json(400, ex) end end @@ -38,13 +38,13 @@ module Invidious::Routes::Subscriptions case action = env.params.query["action"]? when "create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id) + Invidious::Channels::Channels.get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end when "remove_subscriptions" Invidious::Database::Users.unsubscribe_channel(user, channel_id) else - return error_json(400, "Unsupported action #{action}") + return Errors.error_json(400, "Unsupported action #{action}") end if redirect diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index f7909c8e1..99a81fd53 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -24,7 +24,7 @@ module Invidious::Routes::VideoPlayback # Sanity check, to avoid being used as an open proxy if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/) - return error_template(400, "Invalid \"host\" parameter.") + return Errors.error_template(400, "Invalid \"host\" parameter.") end host = "https://#{host}" @@ -83,7 +83,7 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? - playback_statistics = Helpers.get_playback_statistic + playback_statistics = Invidious::Helpers.get_playback_statistic playback_statistics["totalRequests"] += 1 if response.status_code >= 400 @@ -95,7 +95,7 @@ module Invidious::Routes::VideoPlayback if url.includes? "&file=seg.ts" if CONFIG.disabled?("livestreams") - return error_template(403, "Administrator has disabled this endpoint.") + return Errors.error_template(403, "Administrator has disabled this endpoint.") end begin @@ -120,7 +120,7 @@ module Invidious::Routes::VideoPlayback else if query_params["title"]? && CONFIG.disabled?("downloads") || CONFIG.disabled?("dash") - return error_template(403, "Administrator has disabled this endpoint.") + return Errors.error_template(403, "Administrator has disabled this endpoint.") end content_length = nil @@ -195,7 +195,7 @@ module Invidious::Routes::VideoPlayback end end - Helpers.proxy_file(resp, env) + Invidious::Helpers.proxy_file(resp, env) end rescue ex if ex.message != "Error reading socket: Connection reset by peer" @@ -269,11 +269,11 @@ module Invidious::Routes::VideoPlayback # Sanity checks unless id && validate_video_id(id) - return error_template(400, InvalidVideoID.new(id)) + return Errors.error_template(400, InvalidVideoID.new(id)) end if !itag.nil? && (itag <= 0 || itag >= 1000) - return error_template(400, "Invalid itag") + return Errors.error_template(400, "Invalid itag") end region = env.params.query["region"]? @@ -282,15 +282,15 @@ module Invidious::Routes::VideoPlayback title = env.params.query["title"]? if title && CONFIG.disabled?("downloads") - return error_template(403, "Administrator has disabled this endpoint.") + return Errors.error_template(403, "Administrator has disabled this endpoint.") end begin video = get_video(id, region: region) rescue ex : NotFoundException - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex - return error_template(500, ex) + return Errors.error_template(500, ex) end if itag.nil? diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 4bca37531..ef3b80522 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -15,11 +15,11 @@ module Invidious::Routes::Watch id = env.params.query["v"] if env.params.query["v"].empty? - return error_template(400, "Invalid parameters.") + return Errors.error_template(400, "Invalid parameters.") end unless validate_video_id(id) - return error_template(400, InvalidVideoID.new(id)) + return Errors.error_template(400, InvalidVideoID.new(id)) end else return env.redirect "/" @@ -48,10 +48,10 @@ module Invidious::Routes::Watch video = get_video(id, region: params.region) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") - return error_template(404, ex) + return Errors.error_template(404, ex) rescue ex LOGGER.error("get_video: #{id} : #{ex.message}") - return error_template(500, ex) + return Errors.error_template(500, ex) end if preferences.annotations_subscribed && @@ -220,7 +220,7 @@ module Invidious::Routes::Watch if redirect return env.redirect referer else - return error_json(403, "No such user") + return Errors.error_json(403, "No such user") end end @@ -235,12 +235,12 @@ module Invidious::Routes::Watch end begin - validate_request(token, sid, env.request, HMAC_KEY, locale) + Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect - return error_template(400, ex) + return Errors.error_template(400, ex) else - return error_json(400, ex) + return Errors.error_json(400, ex) end end @@ -250,7 +250,7 @@ module Invidious::Routes::Watch when "mark_unwatched" Invidious::Database::Users.mark_unwatched(user, id) else - return error_json(400, "Unsupported action #{action}") + return Errors.error_json(400, "Unsupported action #{action}") end if redirect @@ -264,10 +264,10 @@ module Invidious::Routes::Watch def self.clip(env) clip_id = env.params.url["clip"]? - return error_template(400, "A clip ID is required") if !clip_id + return Errors.error_template(400, "A clip ID is required") if !clip_id response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") - return error_template(400, "Invalid clip ID") if response["error"]? + return Errors.error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s @@ -278,16 +278,16 @@ module Invidious::Routes::Watch return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else - return error_template(404, "The requested clip doesn't exist") + return Errors.error_template(404, "The requested clip doesn't exist") end end def self.download(env) if CONFIG.disabled?("downloads") - return error_template(403, "Administrator has disabled this endpoint.") + return Errors.error_template(403, "Administrator has disabled this endpoint.") end if CONFIG.invidious_companion.present? - return error_template(403, "Downloads should be routed through Companion when present") + return Errors.error_template(403, "Downloads should be routed through Companion when present") end title = env.params.body["title"]? || "" @@ -295,7 +295,7 @@ module Invidious::Routes::Watch selection = env.params.body["download_widget"]? if title.empty? || video_id.empty? || selection.nil? - return error_template(400, "Missing form data") + return Errors.error_template(400, "Missing form data") end download_widget = JSON.parse(selection) @@ -326,7 +326,7 @@ module Invidious::Routes::Watch return Invidious::Routes::VideoPlayback.latest_version(env) else - return error_template(400, "Invalid label or itag") + return Errors.error_template(400, "Invalid label or itag") end end end diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr index 161065e0f..89c5bc18e 100644 --- a/src/invidious/search/ctoken.cr +++ b/src/invidious/search/ctoken.cr @@ -1,32 +1,36 @@ -def produce_channel_search_continuation(ucid, query, page) - if page <= 1 - idx = 0_i64 - else - idx = 30_i64 * (page - 1) - end +module Invidious::Search::CToken + extend self - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "search", - "6:varint" => 1_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "15:base64" => { - "3:varint" => idx, + def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "search", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, + "23:varint" => 0_i64, }, - "23:varint" => 0_i64, + "11:string" => query, + "35:string" => "browse-feed#{ucid}search", }, - "11:string" => query, - "35:string" => "browse-feed#{ucid}search", - }, - } + } - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } - return continuation + return continuation + end end diff --git a/src/invidious/hashtag.cr b/src/invidious/search/hashtag.cr similarity index 89% rename from src/invidious/hashtag.cr rename to src/invidious/search/hashtag.cr index d9d584c96..373dbcffe 100644 --- a/src/invidious/hashtag.cr +++ b/src/invidious/search/hashtag.cr @@ -1,7 +1,7 @@ -module Invidious::Hashtag +module Invidious::Search::Hashtag extend self - def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem) + def fetch_hashtag(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem) cursor = (page - 1) * 60 ctoken = generate_continuation(hashtag, cursor) diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index ecf9aeb31..9da295e31 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -1,45 +1,44 @@ -module Invidious::Search - module Processors - extend self +module Invidious::Search::Processors + extend self - # Regular search (`/search` endpoint) - def regular(query : Query) : Array(SearchItem) - search_params = query.filters.to_yt_params(page: query.page) + # Regular search (`/search` endpoint) + def regular(query : Query) : Array(SearchItem) + search_params = query.filters.to_yt_params(page: query.page) - client_config = YoutubeAPI::ClientConfig.new(region: query.region) - initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) - items, _ = extract_items(initial_data) - return items.reject!(Category) + items, _ = extract_items(initial_data) + return items.reject!(Category) + end + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query : Query) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{query.channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{query.channel}") + response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 + initial_data = Invidious::Helpers.extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(query.channel) if !ucid + else + ucid = query.channel end - # Search a youtube channel - # TODO: clean code, and rely more on YoutubeAPI - def channel(query : Query) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{query.channel}") + continuation = Invidious::Search::CToken.produce_channel_search_continuation(ucid, query.text, query.page) + response_json = YoutubeAPI.browse(continuation) - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{query.channel}") - response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 - initial_data = Helpers.extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(query.channel) if !ucid - else - ucid = query.channel - end + items, _ = extract_items(response_json, "", ucid) + return items.reject!(Category) + end - continuation = produce_channel_search_continuation(ucid, query.text, query.page) - response_json = YoutubeAPI.browse(continuation) + # Search inside of user subscriptions + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" - items, _ = extract_items(response_json, "", ucid) - return items.reject!(Category) - end - - # Search inside of user subscriptions - def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) - view_name = "subscriptions_#{sha256(user.email)}" - - return PG_DB.query_all(" + return PG_DB.query_all(" SELECT id,title,published,updated,ucid,author,length_seconds FROM ( SELECT *, @@ -48,9 +47,8 @@ module Invidious::Search as document FROM #{view_name} ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", - query.text, (query.page - 1) * 20, - as: ChannelVideo - ) - end + query.text, (query.page - 1) * 20, + as: ChannelVideo + ) end end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr deleted file mode 100644 index 622fe5172..000000000 --- a/src/invidious/trending.cr +++ /dev/null @@ -1,48 +0,0 @@ -def fetch_trending(trending_type, region, locale) - region ||= "US" - region = region.upcase - - plid = nil - - browse_id = "" - - case trending_type.try &.downcase - when "gaming" - browse_id = "UCOpNcN46UbXVtpKMrmU4Abg" - params = "Egh0cmVuZGluZw%3D%3D" - when "livestreams" - browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" - params = "EgdsaXZldGFikgEDCKEK" - else - # Livestreams is the default one as Youtube removed - # the aggregated trending page - # https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458 - browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" - params = "EgdsaXZldGFikgEDCKEK" - end - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config) - - items, _ = extract_items(initial_data) - - extracted = [] of SearchItem - - deduplicate = items.size > 1 - - items.each do |itm| - if itm.is_a?(Category) - # Ignore the smaller categories, as they generally contain a sponsored - # channel, which brings a lot of noise on the trending page. - # See: https://github.com/iv-org/invidious/issues/2989 - next if (itm.contents.size < 24 && deduplicate) - - extracted.concat itm.contents.select(SearchItem) - else - extracted << itm - end - end - - # Deduplicate items before returning results - return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid -end diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr index b175c3b92..94c115ac5 100644 --- a/src/invidious/user/captcha.cr +++ b/src/invidious/user/captcha.cr @@ -55,7 +55,7 @@ struct Invidious::User return { question: image, - tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, + tokens: {Invidious::Helpers::Tokens.generate_response(answer, {":login"}, key, use_nonce: true)}, } end end diff --git a/src/invidious/user/converters.cr b/src/invidious/user/converters.cr index dcbf8c534..14bcb75d8 100644 --- a/src/invidious/user/converters.cr +++ b/src/invidious/user/converters.cr @@ -1,12 +1,16 @@ -def convert_theme(theme) - case theme - when "true" - "dark" - when "false" - "light" - when "", nil - nil - else - theme +module Invidious::User::Converters + extend self + + def convert_theme(theme) + case theme + when "true" + "dark" + when "false" + "light" + when "", nil + nil + else + theme + end end end diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr index 3fa468bad..9ce823ae9 100644 --- a/src/invidious/user/exports.cr +++ b/src/invidious/user/exports.cr @@ -15,7 +15,7 @@ struct Invidious::User playlists.each do |playlist| json.object do json.field "title", playlist.title - json.field "description", Helpers.html_to_content(playlist.description_html) + json.field "description", Invidious::Helpers.html_to_content(playlist.description_html) json.field "privacy", playlist.privacy.to_s json.field "videos" do json.array do diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 7c4101cc1..e664016ef 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -43,7 +43,7 @@ struct Invidious::User description = "This is the default description of an imported playlist. Feel Free to change it as you see fit." privacy = PlaylistPrivacy::Private - playlist = create_playlist(title, privacy, user) + playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) # Add each video to the playlist from the body content @@ -117,7 +117,7 @@ struct Invidious::User next if !description next if !privacy - playlist = create_playlist(title, privacy, user) + playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| diff --git a/src/invidious/user/users.cr b/src/invidious/user/users.cr new file mode 100644 index 000000000..ebcce2d51 --- /dev/null +++ b/src/invidious/user/users.cr @@ -0,0 +1,109 @@ +require "crypto/bcrypt/password" + +module Invidious::User::Users + extend self + # Materialized views may not be defined using bound parameters (`$1` as used elsewhere) + MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } + + def create_user(sid, email, password) + password = Crypto::Bcrypt::Password.create(password, cost: 10) + token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) + + user = Invidious::User.new({ + updated: Time.utc, + notifications: [] of String, + subscriptions: [] of String, + email: email, + preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), + password: password.to_s, + token: token, + watched: [] of String, + feed_needs_update: true, + }) + + return user, sid + end + + def get_subscription_feed(user, max_results = 40, page = 1) + limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) + offset = (page - 1) * limit + + notifications = Invidious::Database::Users.select_notifications(user) + view_name = "subscriptions_#{sha256(user.email)}" + + if user.preferences.notifications_only && !notifications.empty? + # Only show notifications + notifications = Invidious::Database::ChannelVideos.select(notifications) + videos = [] of ChannelVideo + + notifications.sort_by!(&.published).reverse! + + case user.preferences.sort + when "alphabetically" + notifications.sort_by!(&.title) + when "alphabetically - reverse" + notifications.sort_by!(&.title).reverse! + when "channel name" + notifications.sort_by!(&.author) + when "channel name - reverse" + notifications.sort_by!(&.author).reverse! + else nil # Ignore + end + else + if user.preferences.latest_only + if user.preferences.unseen_only + # Show latest video from a channel that a user hasn't watched + # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" + + if user.watched.empty? + values = "'{}'" + else + values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" + end + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) + else + # Show latest video from each channel + + videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) + end + + videos.sort_by!(&.published).reverse! + else + if user.preferences.unseen_only + # Only show unwatched + + if user.watched.empty? + values = "'{}'" + else + values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" + end + videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + else + # Sort subscriptions as normal + + videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) + end + end + + case user.preferences.sort + when "published - reverse" + videos.sort_by!(&.published) + when "alphabetically" + videos.sort_by!(&.title) + when "alphabetically - reverse" + videos.sort_by!(&.title).reverse! + when "channel name" + videos.sort_by!(&.author) + when "channel name - reverse" + videos.sort_by!(&.author).reverse! + else nil # Ignore + end + + notifications = Invidious::Database::Users.select_notifications(user) + notifications = videos.select { |v| notifications.includes? v.id } + videos = videos - notifications + end + + return videos, notifications + end +end diff --git a/src/invidious/users.cr b/src/invidious/users.cr deleted file mode 100644 index 65566d207..000000000 --- a/src/invidious/users.cr +++ /dev/null @@ -1,106 +0,0 @@ -require "crypto/bcrypt/password" - -# Materialized views may not be defined using bound parameters (`$1` as used elsewhere) -MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } - -def create_user(sid, email, password) - password = Crypto::Bcrypt::Password.create(password, cost: 10) - token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - - user = Invidious::User.new({ - updated: Time.utc, - notifications: [] of String, - subscriptions: [] of String, - email: email, - preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), - password: password.to_s, - token: token, - watched: [] of String, - feed_needs_update: true, - }) - - return user, sid -end - -def get_subscription_feed(user, max_results = 40, page = 1) - limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) - offset = (page - 1) * limit - - notifications = Invidious::Database::Users.select_notifications(user) - view_name = "subscriptions_#{sha256(user.email)}" - - if user.preferences.notifications_only && !notifications.empty? - # Only show notifications - notifications = Invidious::Database::ChannelVideos.select(notifications) - videos = [] of ChannelVideo - - notifications.sort_by!(&.published).reverse! - - case user.preferences.sort - when "alphabetically" - notifications.sort_by!(&.title) - when "alphabetically - reverse" - notifications.sort_by!(&.title).reverse! - when "channel name" - notifications.sort_by!(&.author) - when "channel name - reverse" - notifications.sort_by!(&.author).reverse! - else nil # Ignore - end - else - if user.preferences.latest_only - if user.preferences.unseen_only - # Show latest video from a channel that a user hasn't watched - # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" - - if user.watched.empty? - values = "'{}'" - else - values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" - end - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) - else - # Show latest video from each channel - - videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) - end - - videos.sort_by!(&.published).reverse! - else - if user.preferences.unseen_only - # Only show unwatched - - if user.watched.empty? - values = "'{}'" - else - values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" - end - videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) - else - # Sort subscriptions as normal - - videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) - end - end - - case user.preferences.sort - when "published - reverse" - videos.sort_by!(&.published) - when "alphabetically" - videos.sort_by!(&.title) - when "alphabetically - reverse" - videos.sort_by!(&.title).reverse! - when "channel name" - videos.sort_by!(&.author) - when "channel name - reverse" - videos.sort_by!(&.author).reverse! - else nil # Ignore - end - - notifications = Invidious::Database::Users.select_notifications(user) - notifications = videos.select { |v| notifications.includes? v.id } - videos = videos - notifications - end - - return videos, notifications -end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 8367fcd78..16fc09e46 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -87,7 +87,7 @@ module Invidious::Videos::Parser # Although technically not a call to /videoplayback the fact that YouTube is returning the # wrong video means that we should count it as a failure. - Helpers.get_playback_statistic["totalRequests"] += 1 + Invidious::Helpers.get_playback_statistic["totalRequests"] += 1 return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), diff --git a/src/invidious/videos.cr b/src/invidious/videos/videos.cr similarity index 100% rename from src/invidious/videos.cr rename to src/invidious/videos/videos.cr diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index ece2efe8a..8ea92a8f7 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -106,7 +106,7 @@
<%=I18n.translate(locale, "timeline_parse_error_show_technical_details")%> - 
<%=get_issue_template(env, item.parse_exception)[1]%>
+ 
<%=Errors.get_issue_template(env, item.parse_exception)[1]%>
<% else %>