adding lzma

CMakeLists.txt

@@ -25,6 +25,7 @@ set(CMAKE_C_FLAGS ${CMAKE_C_FLAGS} "-no-pie")
 
 find_package(lagui REQUIRED)
 find_package(PNG REQUIRED)
+find_package(LibLZMA REQUIRED)
 
 add_compile_options("$<$<C_COMPILER_ID:MSVC>:/std:c11>")
 add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
@@ -63,6 +64,7 @@ if(WIN32)
 endif()
 
 include_directories(
+    ${LIBLZMA_INCLUDE_DIRS}
     ${LAGUI_INCLUDE_DIRS_ALL}
 )
 
@@ -98,6 +100,7 @@ target_compile_definitions(OurPaint PRIVATE "-DOURPAINT_GIT_HASH=\"${OURPAINT_GI
 target_link_libraries(OurPaint
     ${LAGUI_SHARED_LIBS}
     ${PNG_LIBRARY}
+    ${LIBLZMA_LIBRARIES}
 )
 
 SET(INSTALL_EXTRAS

ouroperations.c

@@ -19,6 +19,7 @@
 #include "ourpaint.h"
 #include "png.h"
 #include "lcms2.h"
+#include <lzma.h>
 #include <threads.h>
 #ifdef __linux__
 #include <unistd.h>
@@ -843,14 +844,18 @@ void ourui_AboutContent(laUiList *uil, laPropPack *This, laPropPack *DetachedPro
 void ourui_OurPreference(laUiList *uil, laPropPack *This, laPropPack *DetachedProps, laColumn *UNUSED, int context){
     laColumn* c = laFirstColumn(uil),*cl,*cr; laSplitColumn(uil,c,0.33);cl=laLeftColumn(c,10);cr=laRightColumn(c,0);
     laColumn* crl,*crr; laSplitColumn(uil,cr,0.5); crl=laLeftColumn(cr,0); crr=laRightColumn(cr,0);
-    laUiItem* b,*uiitem;
+    laUiItem* b,*b1,*uiitem;
 
     laShowLabel(uil,c,"Generic:",0,0);
     laShowItemFull(uil,cr,0,"our.preferences.spectral_mode",0,"text=Spectral Mixing (RGBA Canvas only)",0,0)->Flags|=LA_UI_FLAGS_CHECKBOX;
     uiitem=laShowItem(uil,cr,0,"our.preferences.enable_brush_circle");uiitem->Flags|=LA_UI_FLAGS_CHECKBOX;
     b=laOnConditionThat(uil,cr,laPropExpression(&uiitem->PP,""));
-    laShowItemWithLabel(uil,cl,cr,0,"our.preferences.brush_circle_tilt_mode",0,0,0,0,"Show brush direction",0,c)->Flags|=LA_UI_FLAGS_EXPAND;
-    laShowItem(uil,cr,0,"our.preferences.max_recent_files");
+        laShowItemWithLabel(uil,cl,cr,0,"our.preferences.brush_circle_tilt_mode",0,0,0,0,"Show brush direction",0,c)->Flags|=LA_UI_FLAGS_EXPAND;
+        laShowItem(uil,cr,0,"our.preferences.max_recent_files");
+        uiitem=laShowItemWithLabel(uil,cl,crl,0,"our.preferences.compression_method",0,0,0,0,0,0,c);uiitem->Flags|=LA_UI_FLAGS_EXPAND;
+        b1=laOnConditionThat(uil,c,laNot(laPropExpression(&uiitem->PP,"")));{
+            laShowItem(uil,cr,0,"our.preferences.multithread_write")->Flags|=LA_UI_FLAGS_CHECKBOX;
+        }laEndCondition(uil,b1);
     laEndCondition(uil,b);
 
     laShowSeparator(uil,c);
@@ -867,7 +872,6 @@ void ourui_OurPreference(laUiList *uil, laPropPack *This, laPropPack *DetachedPr
     laShowLabel(uil,c,"Canvas:",0,0);
     laShowItem(uil,crl,0,"our.preferences.show_stripes")->Flags|=LA_UI_FLAGS_CHECKBOX;
     laShowItem(uil,crr,0,"our.preferences.show_grid")->Flags|=LA_UI_FLAGS_CHECKBOX;
-    laShowItem(uil,cr,0,"our.preferences.multithread_write")->Flags|=LA_UI_FLAGS_CHECKBOX;
     laShowItem(uil,cr,0,"our.preferences.canvas_default_scale");
     laShowItemWithLabel(uil,cl,cr,0,"our.preferences.default_canvas_type",0,0,0,0,0,0,c)->Flags|=LA_UI_FLAGS_EXPAND;
     laShowItemWithLabel(uil,cl,cr,0,"our.preferences.pigment_display_method",0,0,0,0,0,0,0)->Flags|=LA_UI_FLAGS_EXPAND;
@@ -2544,6 +2548,11 @@ static void _our_png_write(png_structp png_ptr, png_bytep data, png_size_t lengt
     memcpy(&LayerWrite->data[LayerWrite->NextData], data, length);
     LayerWrite->NextData+=length;
 }
+static void _our_lzma_write(OurLayerWrite* LayerWrite, void* data, size_t size){
+    arrEnsureLength(&LayerWrite->data,LayerWrite->NextData+size,&LayerWrite->MaxData,sizeof(unsigned char));
+    memcpy(&LayerWrite->data[LayerWrite->NextData], data, size);
+    LayerWrite->NextData+=size;
+}
 void our_GetImagePigmentDataSimple(int row, int col, OurPigmentData* pd){ //row-=row%2; col-=col%2;
     uint16_t* p0=&Our->ImageBuffer[((int64_t)row*Our->ImageW+col)*4];
     uint16_t* p1=&Our->ImageBuffer[((int64_t)(row+1)*Our->ImageW+col)*4];
@@ -3072,6 +3081,151 @@ cleanup_png_read:
     return result;
 }
 
+#define OUR_LZMA_BUFSIZE (BUFSIZ*16)
+
+void our_ReportLZMAError(char* stage, int ret){
+    static const char* messages[]={
+        "OK",
+        "STREAM_END",
+        "NO_CHECK",
+        "LZMA_UNSUPPORTED_CHECK",
+        "LZMA_GET_CHECK",
+        "LZMA_MEM_ERROR",
+        "LZMA_MEMLIMIT_ERROR",
+        "LZMA_FORMAT_ERROR",
+        "LZMA_OPTIONS_ERROR",
+        "LZMA_DATA_ERROR",
+        "LZMA_BUF_ERROR",
+        "LZMA_PROG_ERROR",
+        "LZMA_SEEK_NEEDED"
+    };
+    TNS_CLAMP(ret,0,12);
+    logPrintNew("LZMA error during %s: %s\n",stage,messages[ret]);
+}
+static int our_InitEncoderLZMA(lzma_stream *strm){
+    lzma_mt mt = {
+        .flags = 0,
+        .block_size = 0,
+        .timeout = 500,
+        .preset = LZMA_PRESET_DEFAULT,
+        .filters = NULL,
+        .check = LZMA_CHECK_CRC64,
+    };
+
+    mt.threads = lzma_cputhreads();
+    if (mt.threads == 0) mt.threads = our_ProcessorCount();
+    if (mt.threads == 0) mt.threads = 1;
+    if (mt.threads > 32) mt.threads =32;
+
+    int ret =lzma_stream_encoder_mt(strm, &mt);
+    if (ret == LZMA_OK) return 1;
+    our_ReportLZMAError("encoder init",ret);
+	return 0;
+}
+static int our_CompressLZMA(void** buf,size_t* bufsize){
+    lzma_stream _strm = LZMA_STREAM_INIT; lzma_stream* strm=&_strm; int res;
+    if(!our_InitEncoderLZMA(strm)) return 0;
+
+    size_t usedsize = 0;
+    lzma_action action = LZMA_RUN;
+    uint8_t* inbuf=Our->ImageBuffer; size_t insize=((size_t)Our->ImageW)*4*Our->ImageH*sizeof(uint16_t);
+    uint8_t outbuf[OUR_LZMA_BUFSIZE];
+    OurLayerWrite LayerWrite={0};
+
+    strm->next_in = NULL; strm->avail_in = 0;
+    strm->next_out = outbuf; strm->avail_out = sizeof(outbuf);
+
+    while (true) {
+        if (strm->avail_in == 0 && usedsize<insize) {
+            strm->next_in = inbuf+usedsize;
+            strm->avail_in = TNS_MIN2(OUR_LZMA_BUFSIZE,insize-usedsize);
+            usedsize+=strm->avail_in;
+            if(usedsize>=insize) action = LZMA_FINISH;
+        }
+
+        lzma_ret ret = lzma_code(strm, action);
+
+        if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
+            size_t write_size = sizeof(outbuf) - strm->avail_out;
+            _our_lzma_write(&LayerWrite,outbuf,write_size);
+            strm->next_out = outbuf; strm->avail_out = sizeof(outbuf);
+        }
+
+        if (ret != LZMA_OK) {
+            if (ret == LZMA_STREAM_END){ *buf=LayerWrite.data; *bufsize=LayerWrite.MaxData; lzma_end(strm); return 1; }
+            our_ReportLZMAError("data encoding",ret);
+            arrFree(&LayerWrite.data,&LayerWrite.MaxData);
+            return 0;
+        }
+        
+        uint64_t inprogress,outprogress; lzma_get_progress(strm,&inprogress,&outprogress);
+        laShowProgress(-1,(real)inprogress/insize);
+    }
+}
+static int our_InitDecoderLZMA(lzma_stream *strm){
+    lzma_mt mt = {
+        .flags = 0,
+        .block_size = 0,
+        .timeout = 500,
+        .preset = LZMA_PRESET_DEFAULT,
+        .memlimit_threading = UINT64_MAX,
+        .memlimit_stop= UINT64_MAX,
+    };
+
+    mt.threads = lzma_cputhreads();
+    if (mt.threads == 0) mt.threads = our_ProcessorCount();
+    if (mt.threads == 0) mt.threads = 1;
+    if (mt.threads > 32) mt.threads =32;
+
+    int ret=lzma_stream_decoder_mt(strm, &mt);
+    if (ret == LZMA_OK) return 1;
+    our_ReportLZMAError("decoder init",ret);
+    return 0;
+}
+static int our_DecompressLZMA(uint8_t* data, size_t datasize){
+    lzma_stream _strm = LZMA_STREAM_INIT; lzma_stream* strm=&_strm; int res;
+    if(!our_InitDecoderLZMA(strm)) return 0;
+
+    size_t usedsize = 0;
+    size_t writtensize = 0;
+    lzma_action action = LZMA_RUN;
+    char* outbuf=Our->ImageBuffer; size_t outsize=((size_t)Our->ImageW)*4*Our->ImageH*sizeof(uint16_t);
+
+    strm->next_in = NULL;
+    strm->avail_in = 0;
+    strm->next_out = outbuf;
+    strm->avail_out = outsize;
+
+    while (true) {
+        if (strm->avail_in == 0 && usedsize<datasize) {
+            strm->next_in = data+usedsize;
+            strm->avail_in = TNS_MIN2(datasize-usedsize,OUR_LZMA_BUFSIZE);
+            usedsize+=strm->avail_in;
+            if(usedsize>=datasize) action = LZMA_FINISH;
+        }
+
+        lzma_ret ret = lzma_code(strm, action);
+
+        if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
+            size_t write_size = sizeof(outbuf) - strm->avail_out;
+            outbuf+=write_size;
+            outsize-=write_size;
+
+            strm->next_out = outbuf;
+            strm->avail_out = outsize;
+        }
+
+        if (ret != LZMA_OK) {
+            if (ret == LZMA_STREAM_END){ lzma_end(strm); return 1; }
+            our_ReportLZMAError("data decoding",ret);
+            return 0;
+        }
+
+        uint64_t inprogress,outprogress; lzma_get_progress(strm,&inprogress,&outprogress);
+        laShowProgress(-1,(real)inprogress/datasize);
+    }
+}
+
 void our_UiToCanvas(laCanvasExtra* ex, laEvent*e, real* x, real *y){
     *x = (real)((real)e->x - (real)(ex->ParentUi->R - ex->ParentUi->L) / 2 - ex->ParentUi->L) * ex->ZoomX + ex->PanX;
     *y = (real)((real)(ex->ParentUi->B - ex->ParentUi->U) / 2 - (real)e->y + ex->ParentUi->U) * ex->ZoomY + ex->PanY;
@@ -4408,6 +4562,7 @@ void ourset_LayerTileStart(OurLayer* l, int* xy){
     Our->TempLoadX = xy[0]; Our->TempLoadY = xy[1];
 }
 void* ourget_LayerImage(OurLayer* l, uint32_t* r_size, int* r_is_copy){
+    if(Our->CompressionMethod){ *r_is_copy=0; return 0; }
     static int LayerCount=0; static int CurrentLayer=0;
     void* buf=0; if(!l->Item.pPrev){ LayerCount=lstCountElements(&Our->Layers); CurrentLayer=0; }
     CurrentLayer++; laShowProgress((real)CurrentLayer/LayerCount,-1);
@@ -4491,6 +4646,8 @@ int ourthread_ExportPNG(OurThreadExportPNGData* data){
     return 0;
 }
 void ourget_LayerImageSegmented(OurLayer* l, int* r_chunks, uint32_t* r_sizes, void** pointers){
+    if(Our->CompressionMethod){ r_chunks=0; return; }
+
     static int LayerCount=0; static int CurrentLayer=0;
     void* buf=0; if(!l->Item.pPrev){ LayerCount=lstCountElements(&Our->Layers); CurrentLayer=0; }
     CurrentLayer++; laShowProgress((real)CurrentLayer/LayerCount,-1);
@@ -4546,6 +4703,37 @@ void ourset_LayerImageSegmentedInfo(OurLayer* l, void* data, int size){
         memcpy(&l->ReadSegmented, data, sizeof(OurLayerImageSegmented));
     }
 }
+void* ourget_LayerImageLZMA(OurLayer* l, uint32_t* r_size, int* r_is_copy){
+    if(Our->CompressionMethod != 1){ *r_is_copy=0; return 0; }
+    
+    static int LayerCount=0; static int CurrentLayer=0;
+    void* buf=0; if(!l->Item.pPrev){ LayerCount=lstCountElements(&Our->Layers); CurrentLayer=0; }
+    CurrentLayer++; laShowProgress((real)CurrentLayer/LayerCount,-1);
+    our_LayerClearEmptyTiles(l);
+    int ensure=our_LayerEnsureImageBuffer(l, 0);
+    if(ensure<=0){ if(!ensure){ our_ShowAllocationError(0); } *r_is_copy=0; return 0; }
+    our_LayerToImageBuffer(l, 0);
+    our_ImageBufferFromNative();
+
+    lzma_stream strm = LZMA_STREAM_INIT;
+    if(our_CompressLZMA(&buf,r_size)){ *r_is_copy=1; return buf; }
+    *r_is_copy=0; return buf;
+}
+void ourset_LayerImageLZMA(OurLayer* l, void* pdata, uint32_t size){
+    if(!pdata) return; char* data=pdata;
+    logPrint("\n    Reading LZMA layer for size %dx%d...",l->ReadSegmented.Width,l->ReadSegmented.Height);
+
+    LA_ACQUIRE_GLES_CONTEXT;
+
+    our_EnsureImageBufferOnRead(l,l->ReadSegmented.Width,l->ReadSegmented.Height,1,Our->TempLoadX, Our->TempLoadY);
+    our_DecompressLZMA(data,size);
+    our_ImageBufferToNative();
+    our_LayerToTexture(l);
+
+    LA_LEAVE_GLES_CONTEXT;
+}
+
+
 
 void ourset_LayerMove(OurLayer* l, int move){ int changed=0;
     if(move<0 && l->Item.pPrev){ lstMoveUp(&Our->Layers, l); changed=1; }
@@ -4809,9 +4997,9 @@ void ourpost_Canvas(void* unused){
     if(Our->CanvasVersion<50){ Our->AlphaMode=0; }
     LA_ACQUIRE_GLES_CONTEXT;
     our_RefreshAllPigmentPreviews();
+    our_AddToRecentFiles(MAIN.ReadingUDF);
     laMarkMemClean(Our->CanvasSaverDummyList.pFirst);
     laMarkMemClean(Our);
-    our_AddToRecentFiles(MAIN.ReadingUDF);
 }
 void ourpost_Brush(OurBrush* brush){
     if(brush->Version<50){ brush->Accumulation=3; brush->PressureAccumulation=1; if(brush->Smudge>0){ brush->SmudgeLifting=0.5; } }
@@ -5335,6 +5523,9 @@ void ourRegisterEverything(){
     laAddEnumItemAs(p,"TRUE","Yes","Enable undo (Restrictions apply)",1,0);
     laAddSubGroup(pc,"recent_files","Recent Files","Recently opened files","our_recent_file",0,0,laui_IdentifierOnly,-1,0,0,0,ourset_RecentFile,0,0,offsetof(OurPaint,RecentFiles),0);
     laAddIntProperty(pc,"max_recent_files","Max Recent Files","The maximum amount of files in the recent file list",0,0,0,20,0,1,5,0,offsetof(OurPaint,MaxRecentFiles),0,0,0,0,0,0,0,0,0,0,0);
+    p=laAddEnumProperty(pc,"compression_method","Compression Method","Compression method used for saving ourpaint files internally",0,0,0,0,0,offsetof(OurPaint,CompressionMethod),0,0,0,0,0,0,0,0,0,0);
+    laAddEnumItemAs(p,"PNG","PNG","Write layers as PNG (fast, bigger file sizes)",0,0);
+    laAddEnumItemAs(p,"LZMA","LZMA","Compress layer data with LZMA (slow, smaller file sizes)",1,0);
 
     pc=laAddPropertyContainer("our_recent_file","Our Recent File","Our recent file item",0,0,sizeof(OurRecentFile),0,0,1);
     laAddStringProperty(pc,"name","Name","Name of the file",0,0,0,0,1,offsetof(OurRecentFile,Name),0,0,0,0,LA_READ_ONLY|LA_AS_IDENTIFIER);
@@ -5563,6 +5754,7 @@ void ourRegisterEverything(){
     laAddRawProperty(pc,"segmented_info","Segmented Info","Image segmented info",0,0,ourget_LayerImageSegmentedInfo,ourset_LayerImageSegmentedInfo,LA_UDF_ONLY);
     p=laAddRawProperty(pc,"image","Image","The image data of this tile",0,0,ourget_LayerImage,ourset_LayerImage,LA_UDF_ONLY);
     laRawPropertyExtraFunctions(p,ourget_LayerImageSegmented,ourget_LayerImageShouldSegment);
+    p=laAddRawProperty(pc,"image_lzma","Image (lzma)","The image data of this tile, compressed with lzma method",0,0,ourget_LayerImageLZMA,ourset_LayerImageLZMA,LA_UDF_ONLY);
     laAddOperatorProperty(pc,"move","Move","Move Layer","OUR_move_layer",0,0);
     laAddOperatorProperty(pc,"remove","Remove","Remove layer","OUR_remove_layer",U'🗴',0);
     laAddOperatorProperty(pc,"merge","Merge","Merge layer","OUR_merge_layer",U'⊻',0);

ourpaint.h

@@ -583,6 +583,7 @@ STRUCTURE(OurPaint){
     int PaletteInColorsPanel;
     int DefaultCanvasType;
     int ToolUndo;
+    int CompressionMethod;
 
     tnsTexture* SmudgeTexture;
     GLuint CanvasShader;         GLuint CanvasProgram;