From 07e97414250827c3b930befa123a4bbd48d24861 Mon Sep 17 00:00:00 2001 From: Juan Linietsky Date: Sun, 23 Aug 2015 20:15:56 -0300 Subject: **WARNING BEFORE PULLING** This push changes the binary and XML formats and bumps the major version to 2.0. As such, files saved in this version WILL NO LONGER WORK IN PREVIOUS VERSIONS. This compatibility breakage with older versions was required in order to properly provide project refactoring tools. If I were you, unless you are brave, I would wait a week or two before pulling, in case of bugs :) Summary of Changes -New Filesystem dock, with filesystem & tree view modes. -New refactoring tools, to change or fix dependencies. -Quick search dialog, to quickly search any file --- core/io/resource_format_binary.cpp | 336 ++++++++++++++++++++++++++++++++++--- 1 file changed, 313 insertions(+), 23 deletions(-) (limited to 'core/io/resource_format_binary.cpp') diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index c6cf631de..60bb8b658 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -31,6 +31,7 @@ #include "globals.h" #include "io/file_access_compressed.h" #include "io/marshalls.h" +#include "os/dir_access.h" //#define print_bl(m_what) print_line(m_what) #define print_bl(m_what) @@ -99,7 +100,9 @@ enum { OBJECT_EMPTY=0, OBJECT_EXTERNAL_RESOURCE=1, OBJECT_INTERNAL_RESOURCE=2, - FORMAT_VERSION=0 + OBJECT_EXTERNAL_RESOURCE_INDEX=3, + FORMAT_VERSION=1, + FORMAT_VERSION_CAN_RENAME_DEPS=1 }; @@ -375,7 +378,7 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant& r_v) { } break; case OBJECT_INTERNAL_RESOURCE: { uint32_t index=f->get_32(); - String path = res_path+"::"+itos(index); + String path = res_path+"::"+itos(index); RES res = ResourceLoader::load(path); if (res.is_null()) { WARN_PRINT(String("Couldn't load resource: "+path).utf8().get_data()); @@ -384,6 +387,7 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant& r_v) { } break; case OBJECT_EXTERNAL_RESOURCE: { + //old file format, still around for compatibility String type = get_unicode_string(); String path = get_unicode_string(); @@ -394,6 +398,10 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant& r_v) { } + if (remaps.find(path)) { + path=remaps[path]; + } + RES res=ResourceLoader::load(path,type); if (res.is_null()) { @@ -401,6 +409,34 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant& r_v) { } r_v=res; + } break; + case OBJECT_EXTERNAL_RESOURCE_INDEX: { + //new file format, just refers to an index in the external list + uint32_t erindex = f->get_32(); + + if (erindex>=external_resources.size()) { + WARN_PRINT("Broken external resource! (index out of size"); + r_v=Variant(); + } else { + + String type = external_resources[erindex].type; + String path = external_resources[erindex].path; + + if (path.find("://")==-1 && path.is_rel_path()) { + // path is relative to file being loaded, so convert to a resource path + path=Globals::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path)); + + } + + RES res=ResourceLoader::load(path,type); + + if (res.is_null()) { + WARN_PRINT(String("Couldn't load resource: "+path).utf8().get_data()); + } + r_v=res; + } + + } break; default: { @@ -628,17 +664,20 @@ Error ResourceInteractiveLoaderBinary::poll(){ if (sstore_32(utf8.length()+1); + f->store_buffer((const uint8_t*)utf8.get_data(),utf8.length()+1); +} + + +static String get_ustring(FileAccess *f) { + + int len = f->get_32(); + Vector str_buf; + str_buf.resize(len); + f->get_buffer((uint8_t*)&str_buf[0],len); + String s; + s.parse_utf8(&str_buf[0]); + return s; +} + String ResourceInteractiveLoaderBinary::get_unicode_string() { int len = f->get_32(); @@ -801,7 +861,7 @@ String ResourceInteractiveLoaderBinary::get_unicode_string() { -void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f,List *p_dependencies) { +void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f,List *p_dependencies,bool p_add_types) { open(p_f); if (error) @@ -814,6 +874,10 @@ void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f,Listpush_back(dep); } @@ -866,7 +930,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { print_bl("minor: "+itos(ver_minor)); print_bl("format: "+itos(ver_format)); - if (ver_formatVERSION_MAJOR) { + if (ver_format>FORMAT_VERSION || ver_major>VERSION_MAJOR) { f->close(); ERR_EXPLAIN("File Format '"+itos(FORMAT_VERSION)+"."+itos(ver_major)+"."+itos(ver_minor)+"' is too new! Please upgrade to a a new engine version: "+local_path); @@ -903,6 +967,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { } //see if the exporter has different set of external resources for more efficient loading + /* String preload_depts = "deps/"+res_path.md5_text(); if (Globals::get_singleton()->has(preload_depts)) { external_resources.clear(); @@ -913,7 +978,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { external_resources[i].path=depts.get_name(i); } print_line(res_path+" - EXTERNAL RESOURCES: "+itos(external_resources.size())); - } + }*/ print_bl("ext resources: "+itos(ext_resources_size)); uint32_t int_resources_size=f->get_32(); @@ -973,7 +1038,7 @@ String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { uint32_t ver_minor=f->get_32(); uint32_t ver_format=f->get_32(); - if (ver_formatVERSION_MAJOR) { + if (ver_format>FORMAT_VERSION || ver_major>VERSION_MAJOR) { f->close(); return ""; @@ -1000,8 +1065,10 @@ ResourceInteractiveLoaderBinary::~ResourceInteractiveLoaderBinary() { } -Ref ResourceFormatLoaderBinary::load_interactive(const String &p_path) { +Ref ResourceFormatLoaderBinary::load_interactive(const String &p_path, Error *r_error) { + if (r_error) + *r_error=ERR_FILE_CANT_OPEN; Error err; FileAccess *f = FileAccess::open(p_path,FileAccess::READ,&err); @@ -1114,7 +1181,7 @@ Error ResourceFormatLoaderBinary::load_import_metadata(const String &p_path, Ref } -void ResourceFormatLoaderBinary::get_dependencies(const String& p_path,List *p_dependencies) { +void ResourceFormatLoaderBinary::get_dependencies(const String& p_path,List *p_dependencies,bool p_add_types) { FileAccess *f = FileAccess::open(p_path,FileAccess::READ); ERR_FAIL_COND(!f); @@ -1123,7 +1190,217 @@ void ResourceFormatLoaderBinary::get_dependencies(const String& p_path,Listlocal_path=Globals::get_singleton()->localize_path(p_path); ria->res_path=ria->local_path; // ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->get_dependencies(f,p_dependencies); + ria->get_dependencies(f,p_dependencies,p_add_types); +} + +Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path,const Map& p_map) { + + +// Error error=OK; + + + FileAccess *f=FileAccess::open(p_path,FileAccess::READ); + ERR_FAIL_COND_V(!f,ERR_CANT_OPEN); + + FileAccess* fw=NULL;//=FileAccess::open(p_path+".depren"); + + String local_path=p_path.get_base_dir(); + + uint8_t header[4]; + f->get_buffer(header,4); + if (header[0]=='R' && header[1]=='S' && header[2]=='C' && header[3]=='C') { + //compressed + FileAccessCompressed *fac = memnew( FileAccessCompressed ); + fac->open_after_magic(f); + f=fac; + + FileAccessCompressed *facw = memnew( FileAccessCompressed ); + facw->configure("RSCC"); + Error err = facw->_open(p_path+".depren",FileAccess::WRITE); + if (err) { + memdelete(fac); + memdelete(facw); + ERR_FAIL_COND_V(err,ERR_FILE_CORRUPT); + } + + fw=facw; + + + } else if (header[0]!='R' || header[1]!='S' || header[2]!='R' || header[3]!='C') { + //not normal + + //error=ERR_FILE_UNRECOGNIZED; + memdelete(f); + ERR_EXPLAIN("Unrecognized binary resource file: "+local_path); + ERR_FAIL_V(ERR_FILE_UNRECOGNIZED); + } else { + fw = FileAccess::open(p_path+".depren",FileAccess::WRITE); + if (!fw) { + memdelete(f); + } + ERR_FAIL_COND_V(!fw,ERR_CANT_CREATE); + } + + bool big_endian = f->get_32(); +#ifdef BIG_ENDIAN_ENABLED + endian_swap = !big_endian; +#else + bool endian_swap = big_endian; +#endif + + bool use_real64 = f->get_32(); + + f->set_endian_swap(big_endian!=0); //read big endian if saved as big endian + fw->store_32(endian_swap); + fw->set_endian_swap(big_endian!=0); + fw->store_32(use_real64); //use real64 + + uint32_t ver_major=f->get_32(); + uint32_t ver_minor=f->get_32(); + uint32_t ver_format=f->get_32(); + + if (ver_formatremove(p_path+".depren"); + memdelete(da); + //fuck it, use the old approach; + + WARN_PRINT(("This file is old, so it can't refactor dependencies, opening and resaving: "+p_path).utf8().get_data()); + + Error err; + f = FileAccess::open(p_path,FileAccess::READ,&err); + if (err!=OK) { + ERR_FAIL_COND_V(err!=OK,ERR_FILE_CANT_OPEN); + } + + Ref ria = memnew( ResourceInteractiveLoaderBinary ); + ria->local_path=Globals::get_singleton()->localize_path(p_path); + ria->res_path=ria->local_path; + ria->remaps=p_map; + // ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); + ria->open(f); + + err = ria->poll(); + + while(err==OK) { + err=ria->poll(); + } + + ERR_FAIL_COND_V(err!=ERR_FILE_EOF,ERR_FILE_CORRUPT); + RES res = ria->get_resource(); + ERR_FAIL_COND_V(!res.is_valid(),ERR_FILE_CORRUPT); + + return ResourceFormatSaverBinary::singleton->save(p_path,res); + } + + if (ver_format>FORMAT_VERSION || ver_major>VERSION_MAJOR) { + + memdelete(f); + memdelete(fw); + ERR_EXPLAIN("File Format '"+itos(FORMAT_VERSION)+"."+itos(ver_major)+"."+itos(ver_minor)+"' is too new! Please upgrade to a a new engine version: "+local_path); + ERR_FAIL_V(ERR_FILE_UNRECOGNIZED); + + } + + fw->store_32( VERSION_MAJOR ); //current version + fw->store_32( VERSION_MINOR ); + fw->store_32( FORMAT_VERSION ); + + save_ustring(fw,get_ustring(f)); //type + + + size_t md_ofs = f->get_pos(); + size_t importmd_ofs = f->get_64(); + fw->store_64(0); //metadata offset + + for(int i=0;i<14;i++) { + fw->store_32(0); + f->get_32(); + } + + //string table + uint32_t string_table_size=f->get_32(); + + fw->store_32(string_table_size); + + for(uint32_t i=0;iget_32(); + fw->store_32(ext_resources_size); + for(uint32_t i=0;iget_pos() - (int64_t)f->get_pos(); + + //internal resources + uint32_t int_resources_size=f->get_32(); + fw->store_32(int_resources_size); + + for(uint32_t i=0;iget_64(); + save_ustring(fw,path); + fw->store_64(offset+size_diff); + } + + //rest of file + uint8_t b = f->get_8(); + while(!f->eof_reached()) { + fw->store_8(b); + b = f->get_8(); + } + + bool all_ok = fw->get_error()==OK; + + fw->seek(md_ofs); + fw->store_64(importmd_ofs+size_diff); + + + memdelete(f); + memdelete(fw); + + if (!all_ok) { + return ERR_CANT_CREATE; + } + + DirAccess *da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + da->remove(p_path); + da->rename(p_path+".depren",p_path); + memdelete(da); + return OK; } @@ -1433,10 +1710,8 @@ void ResourceFormatSaverBinaryInstance::write_variant(const Variant& p_property, } if (res->get_path().length() && res->get_path().find("::")==-1) { - f->store_32(OBJECT_EXTERNAL_RESOURCE); - save_unicode_string(res->get_save_type()); - String path=relative_paths?local_path.path_to_file(res->get_path()):res->get_path(); - save_unicode_string(path); + f->store_32(OBJECT_EXTERNAL_RESOURCE_INDEX); + f->store_32(external_resources[res]); } else { if (!resource_set.has(res)) { @@ -1594,11 +1869,12 @@ void ResourceFormatSaverBinaryInstance::_find_resources(const Variant& p_variant RES res = p_variant.operator RefPtr(); - if (res.is_null()) + if (res.is_null() || external_resources.has(res)) return; if (!p_main && (!bundle_resources ) && res->get_path().length() && res->get_path().find("::") == -1 ) { - external_resources.insert(res); + int idx = external_resources.size(); + external_resources[res]=idx; return; } @@ -1842,10 +2118,18 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path,const RES& p_ // save external resource table f->store_32(external_resources.size()); //amount of external resources - for(Set::Element *E=external_resources.front();E;E=E->next()) { + Vector save_order; + save_order.resize(external_resources.size()); + + for(Map::Element *E=external_resources.front();E;E=E->next()) { + save_order[E->get()]=E->key(); + } - save_unicode_string(E->get()->get_save_type()); - String path = E->get()->get_path(); + for(int i=0;iget_save_type()); + String path = save_order[i]->get_path(); + path=relative_paths?local_path.path_to_file(path):path; save_unicode_string(path); } // save internal resource table @@ -1995,3 +2279,9 @@ void ResourceFormatSaverBinary::get_recognized_extensions(const RES& p_resource, } +ResourceFormatSaverBinary* ResourceFormatSaverBinary::singleton=NULL; + +ResourceFormatSaverBinary::ResourceFormatSaverBinary() { + + singleton=this; +} -- cgit v1.2.3-70-g09d2