class ExtremeZip

Public Class Methods

new() click to toggle source
# File lib/extremezip.zzaqsv.rb, line 19
def initialize
    @wholeCbor = {} # 整个CBOR结构
    @vfsDataList = [] # 数据块压缩块列表
    @filePartCounter = 0 # 文件分块计数器
    @responsePipeList = [] # 任务回复管道列表
    @processIdList = [] # 子进程编号列表。

    @maxSubProcessAmount = Etc.nprocessors # 获取最大的子进程个数

    @dataBlockLength = 33554432 # 数据块单元长度, 32MiB

    @clipDownloader = VictoriaFresh.new # 创建下载器。

    @clipDownloader.diskFlush = true # 向磁盘写入缓存
    @clipDownloader.diskMultiFile = true # 写多个磁盘文件
    @clipDownloader.diskFileName = 'victoriafreshdata.v.' # 磁盘文件名前缀
    @clipDownloader.diskFlushSize = @dataBlockLength # 磁盘文件大小
end

Public Instance Methods

addBasicFileInformation() click to toggle source

加入基本文件信息

# File lib/extremezip.zzaqsv.rb, line 49
def addBasicFileInformation
    @wholeCbor['version'] = 68 # 文件格式版本号

    uuid = UUID.new # 获取生成器
    @wholeCbor['uuid'] = uuid.generate # 指定本个压缩包的唯一编号
end
compressInSubProcess(currentBlockData, currentResponsePipe) click to toggle source

在子进程中具体执行的压缩代码

# File lib/extremezip.zzaqsv.rb, line 159
def compressInSubProcess(currentBlockData, currentResponsePipe)
    checkMemoryUsage(115) # Debug
    currentBlockDataToCompress = currentBlockData # 读取数据块

    currentCompressedVfsData = LZMA.compress(currentBlockDataToCompress) # 压缩当前块

    checkMemoryUsage(120)
    puts("compressed data length: #{currentCompressedVfsData.bytesize}") # Debug.

    currentResponsePipe.put currentCompressedVfsData # 将压缩后的数据块写入到回复管道中

    checkMemoryUsage(125)
    puts("finished #{Process.pid}") # Debug
end
compressVfsMenu(victoriaFresh) click to toggle source

压缩目录数据。

# File lib/extremezip.zzaqsv.rb, line 39
def compressVfsMenu(victoriaFresh)
    replyByteArray = victoriaFresh.to_cbor # #打包成字节数组。

    # 压缩目录数据并放入CBOR:
    compressedVfsMenu = LZMA.compress(replyByteArray) # 压缩目录数据

    @wholeCbor['vfsMenu'] = compressedVfsMenu # 加入目录
end
exz(rootPath) click to toggle source

压缩

# File lib/extremezip.zzaqsv.rb, line 57
def exz(rootPath)
    victoriaFresh, = @clipDownloader.checkOnce(rootPath) # 打包该目录树。

    @filePartAmount = @clipDownloader.currentDiskFlushSuffix # 获取文件个数

    compressVfsMenu(victoriaFresh) # 压缩目录数据。

    addBasicFileInformation # 加入基本文件信息

    processIdList, responsePipeList = launchSubProcesses # 启动子进程。

    receiveCompressedVfsDataList(processIdList, responsePipeList) # 接收压缩后的数据块列表

    checkMemoryUsage(155)
    @wholeCbor['vfsDataList'] = @vfsDataList # 加入数据

    wholeFileContent = 'exz' + "\0" + @wholeCbor.to_cbor # 追加CBOR字节数组

    # 写入文件:
    writeFile(wholeFileContent, victoriaFresh) # 写入文件内容
end
launchSubProcesses() click to toggle source

启动子进程。

# File lib/extremezip.zzaqsv.rb, line 147
def launchSubProcesses

    #while ((@filePartCounter < @filePartAmount) && (true)) # 未处理完毕,并且未达到最大子进程个数
    while ((@filePartCounter < @filePartAmount) && (@filePartCounter<@maxSubProcessAmount)) # 未处理完毕,并且未达到最大子进程个数
        currentResponsePipe, p1 = schedule1Block(@filePartCounter) # 计划一个块的压缩计算

    end # while processDataLength < victoriaFreshData.byte_size do #未处理完毕

    [@processIdList, @responsePipeList]
end
readBlockFile(filePartCounter) click to toggle source

读取块文件内容

# File lib/extremezip.zzaqsv.rb, line 107
def readBlockFile(filePartCounter)
    currentBlockFile = File.new(@clipDownloader.diskFileName + filePartCounter.to_s, 'rb') # 打开文件

    currentBlockData = currentBlockFile.read # 读取全部内容

    currentBlockFile.close # 关闭文件

    File.delete(currentBlockFile) # 删除数据块文件

    currentBlockData
end
receiveCompressedVfsDataList(processIdList, responsePipeList) click to toggle source

接收压缩后的数据块列表

# File lib/extremezip.zzaqsv.rb, line 87
def receiveCompressedVfsDataList(processIdList, responsePipeList)
    processCounter = 0 # 子进程计数器
    
    while (processCounter<@filePartAmount) # 并不是所有分块都被处理完毕了。
        currentSubProcess=processIdList[processCounter] # 获取子进程对象
    #processIdList.each do |currentSubProcess|
        compressed = receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) # 从子进程中读取数据,并终止子进程

        @vfsDataList << compressed # 加入数据块列表中
        checkMemoryUsage(150)

        processCounter += 1 # 子进程计数
        
        if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理
            schedule1Block(@filePartCounter) # 再启动一个子进程
        end # if (@filePartCounter<@filePartAmount) # 还有一些分块尚未交给子进程进行处理
    end # processIdList.each do |currentSubProcess|
end
receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter) click to toggle source

从子进程中读取数据,并终止子进程

# File lib/extremezip.zzaqsv.rb, line 175
def receiveFromSubProcess(currentSubProcess, responsePipeList, processCounter)
    puts("waiting #{currentSubProcess}") # Debug
    checkMemoryUsage(140)

    currentResponsePipe = responsePipeList[processCounter] # 任务回复管道

    currentCompressedVfsDataFromSubProcess = currentResponsePipe.get # 读取压缩后数据
    checkMemoryUsage(145)

    Process.waitpid(currentSubProcess) # 等待该个子进程

    currentCompressedVfsDataFromSubProcess
end
schedule1Block(filePartCounter) click to toggle source

计划一个块的压缩计算

# File lib/extremezip.zzaqsv.rb, line 120
def schedule1Block(filePartCounter)
    currentBlockData = readBlockFile(filePartCounter) # 读取块文件内容

    # currentTaskPipe = Cod.pipe # 任务分配管道
    currentResponsePipe = Cod.pipe # 任务回复管道

    puts("forking sub process, file part counter: #{filePartCounter}") # Debug.

    p1 = fork do # 复制出子进程
        compressInSubProcess(currentBlockData, currentResponsePipe) # 在子进程中具体执行的压缩代码
    end # p1 = fork do #复制出子进程

    # processDataLength += @dataBlockLength # 计数
    checkMemoryUsage(130)

                # 记录管道:
        # taskPipeList << currentTaskPipe
        @responsePipeList << currentResponsePipe # 记录回复管道

        @processIdList << p1 # 记录到子进程列表中

        @filePartCounter += 1 # 计数

    [currentResponsePipe, p1]
end
writeFile(wholeFileContent, victoriaFresh) click to toggle source

写入文件内容

# File lib/extremezip.zzaqsv.rb, line 80
def writeFile(wholeFileContent, victoriaFresh)
    extremeZipOutputFile = File.new("#{victoriaFresh['name']}.exz", 'wb') # 创建文件
    extremeZipOutputFile.syswrite(wholeFileContent) # 写入文件
    extremeZipOutputFile.close # 关闭文件
end