github/hubproxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化离线下载镜像的实现

Browse Source
This commit is contained in:
user123
2026-02-02 06:12:31 +08:00
parent 3917b2503a
commit 23dd077f5d
4 changed files with 342 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
src/handlers/imagetar.go
View File

@@ -5,12 +5,15 @@ import (
"compress/gzip"
"context"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"sort"
"strings"
"sync"
@@ -115,6 +118,15 @@ func getUserID(c *gin.Context) string {
return "ip:" + hex.EncodeToString(hash[:8])
}
func getClientIdentity(c *gin.Context) (string, string) {
ip := c.ClientIP()
userAgent := c.GetHeader("User-Agent")
if userAgent == "" {
userAgent = "unknown"
}
return ip, userAgent
}
var (
singleImageDebouncer *DownloadDebouncer
batchImageDebouncer *DownloadDebouncer
@@ -126,6 +138,98 @@ func InitDebouncer() {
batchImageDebouncer = NewDownloadDebouncer(60 * time.Second)
}
type BatchDownloadRequest struct {
Images []string
Platform string
UseCompressedLayers bool
}
type SingleDownloadRequest struct {
Image string
Platform string
UseCompressedLayers bool
}
type tokenEntry[T any] struct {
Request T
ExpiresAt time.Time
IP string
UserAgent string
}
type tokenStore[T any] struct {
mu sync.RWMutex
entries map[string]tokenEntry[T]
}
const downloadTokenTTL = 2 * time.Minute
const downloadTokenMaxEntries = 2000
func newTokenStore[T any]() *tokenStore[T] {
return &tokenStore[T]{
entries: make(map[string]tokenEntry[T]),
}
}
func (s *tokenStore[T]) create(req T, ip, userAgent string) (string, error) {
tokenBytes := make([]byte, 32)
if _, err := rand.Read(tokenBytes); err != nil {
return "", err
}
token := base64.RawURLEncoding.EncodeToString(tokenBytes)
now := time.Now()
entry := tokenEntry[T]{
Request: req,
ExpiresAt: now.Add(downloadTokenTTL),
IP: ip,
UserAgent: userAgent,
}
s.mu.Lock()
s.cleanup(now)
if len(s.entries) >= downloadTokenMaxEntries {
s.mu.Unlock()
return "", fmt.Errorf("令牌过多，请稍后再试")
}
s.entries[token] = entry
s.mu.Unlock()
return token, nil
}
func (s *tokenStore[T]) consume(token, ip, userAgent string) (T, bool) {
var empty T
now := time.Now()
s.mu.Lock()
defer s.mu.Unlock()
entry, exists := s.entries[token]
if !exists {
return empty, false
}
if now.After(entry.ExpiresAt) {
delete(s.entries, token)
return empty, false
}
if entry.IP != ip || entry.UserAgent != userAgent {
delete(s.entries, token)
return empty, false
}
delete(s.entries, token)
return entry.Request, true
}
func (s *tokenStore[T]) cleanup(now time.Time) {
for token, entry := range s.entries {
if now.After(entry.ExpiresAt) {
delete(s.entries, token)
}
}
}
var batchDownloadTokens = newTokenStore[BatchDownloadRequest]()
var singleDownloadTokens = newTokenStore[SingleDownloadRequest]()
// ImageStreamer 镜像流式下载器
type ImageStreamer struct {
concurrency int
@@ -209,6 +313,17 @@ func (is *ImageStreamer) getImageDescriptorWithPlatform(ref name.Reference, opti
return remote.Get(ref, options...)
}
func setDownloadHeaders(c *gin.Context, filename string, compressed bool) {
c.Header("Content-Type", "application/octet-stream")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
c.Header("Cache-Control", "no-store")
c.Header("Pragma", "no-cache")
c.Header("Expires", "0")
if compressed {
c.Header("Content-Encoding", "gzip")
}
}
// StreamImageToGin 流式响应到Gin
func (is *ImageStreamer) StreamImageToGin(ctx context.Context, imageRef string, c *gin.Context, options *StreamOptions) error {
if options == nil {
@@ -216,12 +331,7 @@ func (is *ImageStreamer) StreamImageToGin(ctx context.Context, imageRef string,
}
filename := strings.ReplaceAll(imageRef, "/", "_") + ".tar"
c.Header("Content-Type", "application/octet-stream")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
if options.Compression {
c.Header("Content-Encoding", "gzip")
}
setDownloadHeaders(c, filename, options.Compression)
return is.StreamImageToWriter(ctx, imageRef, c.Writer, options)
}
@@ -579,6 +689,7 @@ func InitImageTarRoutes(router *gin.Engine) {
{
imageAPI.GET("/download/:image", handleDirectImageDownload)
imageAPI.GET("/info/:image", handleImageInfo)
imageAPI.GET("/batch", handleSimpleBatchDownload)
imageAPI.POST("/batch", handleSimpleBatchDownload)
}
}
@@ -607,27 +718,64 @@ func handleDirectImageDownload(c *gin.Context) {
return
}
userID := getUserID(c)
contentKey := generateContentFingerprint([]string{imageRef}, platform)
if c.Query("mode") == "prepare" {
userID := getUserID(c)
contentKey := generateContentFingerprint([]string{imageRef}, platform)
if !singleImageDebouncer.ShouldAllow(userID, contentKey) {
c.JSON(http.StatusTooManyRequests, gin.H{
"error": "请求过于频繁，请稍后再试",
"retry_after": 5,
})
if !singleImageDebouncer.ShouldAllow(userID, contentKey) {
c.JSON(http.StatusTooManyRequests, gin.H{
"error": "请求过于频繁，请稍后再试",
"retry_after": 5,
})
return
}
ip, userAgent := getClientIdentity(c)
token, err := singleDownloadTokens.create(SingleDownloadRequest{
Image: imageRef,
Platform: platform,
UseCompressedLayers: useCompressed,
}, ip, userAgent)
if err != nil {
c.JSON(http.StatusTooManyRequests, gin.H{"error": err.Error()})
return
}
downloadURL := fmt.Sprintf("/api/image/download/%s?token=%s", imageParam, token)
if tag != "" {
downloadURL = downloadURL + "&tag=" + url.QueryEscape(tag)
}
c.JSON(http.StatusOK, gin.H{"download_url": downloadURL})
return
}
token := c.Query("token")
if token == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "缺少下载令牌"})
return
}
ip, userAgent := getClientIdentity(c)
req, ok := singleDownloadTokens.consume(token, ip, userAgent)
if !ok {
c.JSON(http.StatusBadRequest, gin.H{"error": "无效或过期的下载令牌"})
return
}
if req.Image != imageRef {
c.JSON(http.StatusBadRequest, gin.H{"error": "下载令牌与镜像不匹配"})
return
}
options := &StreamOptions{
Platform: platform,
Platform: req.Platform,
Compression: false,
UseCompressedLayers: useCompressed,
UseCompressedLayers: req.UseCompressedLayers,
}
ctx := c.Request.Context()
log.Printf("下载镜像: %s (平台: %s)", imageRef, formatPlatformText(platform))
log.Printf("下载镜像: %s (平台: %s)", req.Image, formatPlatformText(req.Platform))
if err := globalImageStreamer.StreamImageToGin(ctx, imageRef, c, options); err != nil {
if err := globalImageStreamer.StreamImageToGin(ctx, req.Image, c, options); err != nil {
log.Printf("镜像下载失败: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "镜像下载失败: " + err.Error()})
return
@@ -636,6 +784,51 @@ func handleDirectImageDownload(c *gin.Context) {
// handleSimpleBatchDownload 处理批量下载
func handleSimpleBatchDownload(c *gin.Context) {
if c.Request.Method == http.MethodGet {
token := c.Query("token")
if token == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "缺少下载令牌"})
return
}
ip, userAgent := getClientIdentity(c)
req, ok := batchDownloadTokens.consume(token, ip, userAgent)
if !ok {
c.JSON(http.StatusBadRequest, gin.H{"error": "无效或过期的下载令牌"})
return
}
if len(req.Images) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "镜像列表不能为空"})
return
}
options := &StreamOptions{
Platform: req.Platform,
Compression: false,
UseCompressedLayers: req.UseCompressedLayers,
}
ctx := c.Request.Context()
log.Printf("批量下载 %d 个镜像 (平台: %s)", len(req.Images), formatPlatformText(req.Platform))
filename := fmt.Sprintf("batch_%d_images.tar", len(req.Images))
setDownloadHeaders(c, filename, options.Compression)
if err := globalImageStreamer.StreamMultipleImages(ctx, req.Images, c.Writer, options); err != nil {
log.Printf("批量镜像下载失败: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "批量镜像下载失败: " + err.Error()})
return
}
return
}
if c.Query("mode") != "prepare" {
c.JSON(http.StatusBadRequest, gin.H{"error": "只支持prepare模式"})
return
}
var req struct {
Images []string `json:"images" binding:"required"`
Platform string `json:"platform"`
@@ -682,25 +875,19 @@ func handleSimpleBatchDownload(c *gin.Context) {
useCompressed = *req.UseCompressedLayers
}
options := &StreamOptions{
batchReq := BatchDownloadRequest{
Images: req.Images,
Platform: req.Platform,
Compression: false,
UseCompressedLayers: useCompressed,
}
ctx := c.Request.Context()
log.Printf("批量下载 %d 个镜像 (平台: %s)", len(req.Images), formatPlatformText(req.Platform))
filename := fmt.Sprintf("batch_%d_images.tar", len(req.Images))
c.Header("Content-Type", "application/octet-stream")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
if err := globalImageStreamer.StreamMultipleImages(ctx, req.Images, c.Writer, options); err != nil {
log.Printf("批量镜像下载失败: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "批量镜像下载失败: " + err.Error()})
ip, userAgent := getClientIdentity(c)
token, err := batchDownloadTokens.create(batchReq, ip, userAgent)
if err != nil {
c.JSON(http.StatusTooManyRequests, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"download_url": fmt.Sprintf("/api/image/batch?token=%s", token)})
}
// handleImageInfo 处理镜像信息查询

2
src/handlers/search.go
View File

@@ -251,7 +251,7 @@ func searchDockerHubWithDepth(ctx context.Context, query string, page, pageSize
}
return nil, fmt.Errorf("未找到相关镜像")
case http.StatusBadGateway, http.StatusServiceUnavailable:
return nil, fmt.Errorf("Docker Hub服务暂时不可用，请稍后重试")
return nil, fmt.Errorf("docker hub 服务暂时不可用，请稍后重试")
default:
return nil, fmt.Errorf("请求失败: 状态码=%d, 响应=%s", resp.StatusCode, string(body))
}

151
src/public/images.html vendored
View File

@@ -728,7 +728,7 @@
}
}
function buildDownloadUrl(imageName, platform = '', useCompressed = true) {
function buildDownloadUrl(imageName, platform = '', useCompressed = true, mode = '') {
const encodedImage = imageName.replace(/\//g, '_');
let url = `/api/image/download/${encodedImage}`;
@@ -737,6 +737,9 @@
params.append('platform', platform.trim());
}
params.append('compressed', useCompressed.toString());
if (mode) {
params.append('mode', mode);
}
if (params.toString()) {
url += '?' + params.toString();
@@ -745,7 +748,65 @@
return url;
}
document.getElementById('singleForm').addEventListener('submit', function(e) {
function buildInfoUrl(imageName) {
const encodedImage = imageName.replace(/\//g, '_');
return `/api/image/info/${encodedImage}`;
}
async function preflightImageDownload(imageName) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 8000);
try {
const response = await fetch(buildInfoUrl(imageName), {
method: 'GET',
headers: {
'Accept': 'application/json'
},
cache: 'no-store',
signal: controller.signal
});
const contentType = response.headers.get('Content-Type') || '';
let payload = null;
if (contentType.includes('application/json')) {
payload = await response.json();
}
if (!response.ok) {
return { ok: false, error: (payload && payload.error) ? payload.error : '镜像预检失败' };
}
if (payload && payload.success === false) {
return { ok: false, error: payload.error || '镜像预检失败' };
}
return { ok: true };
} catch (error) {
if (error.name === 'AbortError') {
return { ok: false, error: '预检超时，请稍后重试' };
}
return { ok: false, error: '网络错误: ' + error.message };
} finally {
clearTimeout(timeoutId);
}
}
async function preflightImages(images) {
const uniqueImages = Array.from(new Set(images));
const results = await Promise.allSettled(uniqueImages.map((imageName) => preflightImageDownload(imageName)));
for (let i = 0; i < results.length; i++) {
const result = results[i];
if (result.status === 'rejected') {
return { ok: false, error: '预检失败，请稍后重试' };
}
if (!result.value.ok) {
return { ok: false, error: result.value.error || '预检失败' };
}
}
return { ok: true };
}
document.getElementById('singleForm').addEventListener('submit', async function(e) {
e.preventDefault();
const imageName = document.getElementById('imageInput').value.trim();
@@ -760,20 +821,50 @@
hideStatus('singleStatus');
setButtonLoading('downloadBtn', 'downloadText', 'downloadLoading', true);
const downloadUrl = buildDownloadUrl(imageName, platform, useCompressed);
const link = document.createElement('a');
link.href = downloadUrl;
link.download = '';
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
const platformText = platform ? ` (${platform})` : '';
showStatus('singleStatus', `开始下载 ${imageName}${platformText}`, 'success');
setButtonLoading('downloadBtn', 'downloadText', 'downloadLoading', false);
showStatus('singleStatus', '正在准备下载...', 'success');
const preflightResult = await preflightImages([imageName]);
if (!preflightResult.ok) {
showStatus('singleStatus', preflightResult.error, 'error');
setButtonLoading('downloadBtn', 'downloadText', 'downloadLoading', false);
return;
}
const prepareUrl = buildDownloadUrl(imageName, platform, useCompressed, 'prepare');
try {
const response = await fetch(prepareUrl, {
method: 'GET',
headers: {
'Accept': 'application/json'
},
cache: 'no-store'
});
if (response.ok) {
const data = await response.json();
if (!data || !data.download_url) {
showStatus('singleStatus', '下载地址生成失败', 'error');
return;
}
const link = document.createElement('a');
link.href = data.download_url;
link.download = '';
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
const platformText = platform ? ` (${platform})` : '';
showStatus('singleStatus', `开始下载 ${imageName}${platformText}`, 'success');
} else {
const error = await response.json();
showStatus('singleStatus', error.error || '下载失败', 'error');
}
} catch (error) {
showStatus('singleStatus', '网络错误: ' + error.message, 'error');
} finally {
setButtonLoading('downloadBtn', 'downloadText', 'downloadLoading', false);
}
});
document.getElementById('batchForm').addEventListener('submit', async function(e) {
@@ -808,9 +899,16 @@
hideStatus('batchStatus');
setButtonLoading('batchDownloadBtn', 'batchDownloadText', 'batchDownloadLoading', true);
showStatus('batchStatus', '正在准备下载...', 'success');
const preflightResult = await preflightImages(images);
if (!preflightResult.ok) {
showStatus('batchStatus', preflightResult.error, 'error');
setButtonLoading('batchDownloadBtn', 'batchDownloadText', 'batchDownloadLoading', false);
return;
}
try {
const response = await fetch('/api/image/batch', {
const response = await fetch('/api/image/batch?mode=prepare', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -819,24 +917,19 @@
});
if (response.ok) {
const contentDisposition = response.headers.get('Content-Disposition');
let filename = `batch_${images.length}_images.tar`;
if (contentDisposition) {
const matches = contentDisposition.match(/filename="(.+)"/);
if (matches) filename = matches[1];
const data = await response.json();
if (!data || !data.download_url) {
showStatus('batchStatus', '下载地址生成失败', 'error');
return;
}
const blob = await response.blob();
const url = window.URL.createObjectURL(blob);
const url = data.download_url;
const link = document.createElement('a');
link.href = url;
link.download = filename;
link.style.display = 'none';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
window.URL.revokeObjectURL(url);
const platformText = platform ? ` (${platform})` : '';
showStatus('batchStatus', `开始下载 ${images.length} 个镜像${platformText}`, 'success');
@@ -872,4 +965,4 @@
initMobileMenu();
</script>
</body>
</html>
</html>

14
src/public/search.html vendored
View File

@@ -778,16 +778,10 @@
</div>
</div>
<div class="tag-list" id="tagList">
<div class="pagination" id="tagPagination" style="display: none;">
<button id="tagPrevPage" disabled>上一页</button>
<button id="tagNextPage" disabled>下一页</button>
</div>
</div>
<div class="tag-list" id="tagList"></div>
</div>
<div id="toast"></div>
<script>
const formatUtils = {
formatNumber(num) {
@@ -1047,7 +1041,6 @@
}
// 分页更新函数
const updateSearchPagination = () => updatePagination();
const updateTagPagination = () => updatePagination({
currentPage: currentTagPage,
totalPages: totalTagPages,
@@ -1217,9 +1210,6 @@
const currentNamespace = namespace || repoInfo.namespace;
const currentName = name || repoInfo.name;
// 调试日志
console.log(`loadTagPage: namespace=${currentNamespace}, name=${currentName}, page=${currentTagPage}`);
if (!currentNamespace || !currentName) {
showToast('命名空间和镜像名称不能为空');
return;
@@ -1488,4 +1478,4 @@
</script>
</main>
</body>
</html>
</html>