diff -ruNp -X excludefiles trunk.409/action.c localwork/action.c --- trunk.409/action.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/action.c 2009-11-25 12:30:30.000000000 +0100 @@ -51,10 +51,8 @@ void csync_schedule_commands(const char continue; found_matching_pattern: for (c=a->command; c; c=c->next) - SQL("Add action to database", - "INSERT INTO action (filename, command, logfile) " - "VALUES ('%s', '%s', '%s')", url_encode(filename), - url_encode(c->command), url_encode(a->logfile)); + csdbAddActionFilename_Command_Logfile_( + url_encode(filename), url_encode(c->command), url_encode(a->logfile)); } } } @@ -67,12 +65,7 @@ void csync_run_single_command(const char struct textlist *tl = 0, *t; pid_t pid; - SQL_BEGIN("Checking for removed files", - "SELECT filename from action WHERE command = '%s' " - "and logfile = '%s'", command, logfile) - { - textlist_add(&tl, SQL_V(0), 0); - } SQL_END; + tl=csdbGetActionCommand_Logfile_(command,logfile); mark = strstr(command_clr, "%%"); if ( !mark ) { @@ -117,10 +110,7 @@ void csync_run_single_command(const char csync_fatal("ERROR: Waitpid returned error %s.\n", strerror(errno)); for (t = tl; t != 0; t = t->next) - SQL("Remove action entry", - "DELETE FROM action WHERE command = '%s' " - "and logfile = '%s' and filename = '%s'", - command, logfile, t->value); + csdbDeleteActionFilename_Command_Logfile_(t->value,command,logfile); textlist_free(tl); } @@ -129,11 +119,7 @@ void csync_run_commands() { struct textlist *tl = 0, *t; - SQL_BEGIN("Checking for sceduled commands", - "SELECT command, logfile FROM action GROUP BY command, logfile") - { - textlist_add2(&tl, SQL_V(0), SQL_V(1), 0); - } SQL_END; + tl=csdbGetActionCommands(); for (t = tl; t != 0; t = t->next) csync_run_single_command(t->value, t->value2); diff -ruNp -X excludefiles trunk.409/AUTHORS localwork/AUTHORS --- trunk.409/AUTHORS 2009-11-24 18:04:53.000000000 +0100 +++ localwork/AUTHORS 2009-11-24 18:12:09.000000000 +0100 @@ -1,2 +1,4 @@ LINBIT Information Technologies GmbH Clifford Wolf +IPC constructs for powerinternet.eu +database daemon constructs for telegraaf.nl diff -ruNp -X excludefiles trunk.409/cfgfile_parser.y localwork/cfgfile_parser.y --- trunk.409/cfgfile_parser.y 2009-11-24 18:04:53.000000000 +0100 +++ localwork/cfgfile_parser.y 2009-11-25 14:24:09.000000000 +0100 @@ -33,6 +33,9 @@ struct csync_nossl *csync_nossl = 0; int csync_ignore_uid = 0; int csync_ignore_gid = 0; int csync_ignore_mod = 0; +int csync_db_split = 0; +char *csync_tempdir = NULL; +char *csync_csdbsocket = NULL; #ifdef __CYGWIN__ int csync_lowercyg_disable = 0; @@ -310,6 +313,21 @@ static void set_action_dolocal_only() csync_group->action->do_local_only = 1; } +static void set_tempdir(const char *tempdir) +{ + csync_tempdir=strdup(tempdir); +} + +static void set_csdbsocket(const char *csdbsocket) +{ + csync_csdbsocket=strdup(csdbsocket); +} + +static void set_db_split(void) +{ + csync_db_split=1; +} + static void new_prefix(const char *pname) { struct csync_prefix *p = @@ -402,6 +420,9 @@ static void disable_cygwin_lowercase_hac %token TK_ACTION TK_PATTERN TK_EXEC TK_DOLOCAL TK_LOGFILE TK_NOCYGLOWER %token TK_PREFIX TK_ON TK_COLON TK_POPEN TK_PCLOSE %token TK_BAK_DIR TK_BAK_GEN TK_DOLOCALONLY +%token TK_TEMPDIR +%token TK_CSDBSOCKET +%token TK_DBSPLIT %token TK_STRING %% @@ -419,6 +440,12 @@ block: { } | TK_NOSSL TK_STRING TK_STRING TK_STEND { new_nossl($2, $3); } +| TK_DBSPLIT TK_STEND + { set_db_split(); } +| TK_CSDBSOCKET TK_STRING TK_STEND + { set_csdbsocket($2); } +| TK_TEMPDIR TK_STRING TK_STEND + { set_tempdir($2); } | TK_IGNORE ignore_list TK_STEND | TK_NOCYGLOWER TK_STEND { disable_cygwin_lowercase_hack(); } diff -ruNp -X excludefiles trunk.409/cfgfile_scanner.l localwork/cfgfile_scanner.l --- trunk.409/cfgfile_scanner.l 2009-11-24 18:04:53.000000000 +0100 +++ localwork/cfgfile_scanner.l 2009-11-25 14:03:06.000000000 +0100 @@ -61,6 +61,9 @@ int include_stack_ptr = 0; "prefix" { return TK_PREFIX; } "on" { return TK_ON; } +"tempdir" { return TK_TEMPDIR; } +"csdb-socket" { return TK_CSDBSOCKET; } +"split-databases" { return TK_DBSPLIT; } "backup-directory" { return TK_BAK_DIR; } "backup-generations" { return TK_BAK_GEN; } diff -ruNp -X excludefiles trunk.409/check.c localwork/check.c --- trunk.409/check.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/check.c 2009-11-24 18:22:30.000000000 +0100 @@ -79,9 +79,7 @@ check_failed: void csync_hint(const char *file, int recursive) { - SQL("Adding Hint", - "INSERT INTO hint (filename, recursive) " - "VALUES ('%s', %d)", url_encode(file), recursive); + csdbAddHintFilename_Flags_(url_encode(file),recursive); } void csync_mark(const char *file, const char *thispeer, const char *peerfilter) @@ -100,12 +98,9 @@ void csync_mark(const char *file, const csync_debug(1, "Marking file as dirty: %s\n", file); for (pl_idx=0; pl[pl_idx].peername; pl_idx++) if (!peerfilter || !strcmp(peerfilter, pl[pl_idx].peername)) - SQL("Marking File Dirty", - "%s INTO dirty (filename, force, myname, peername) " - "VALUES ('%s', %s, '%s', '%s')", - csync_new_force ? "REPLACE" : "INSERT", + csdbAddDirtyFilename_Flags_Myname_Peername_( url_encode(file), - csync_new_force ? "1" : "0", + csync_new_force?DIRTY_FORCE:0, url_encode(pl[pl_idx].myname), url_encode(pl[pl_idx].peername)); @@ -142,39 +137,43 @@ int csync_check_pure(const char *filenam void csync_check_del(const char *file, int recursive, int init_run) { - char *where_rec = ""; struct textlist *tl = 0, *t; + struct textlist *ftl; struct stat st; + char *encoded_file=NULL; + if(strcmp(file,"/")) { + encoded_file=url_encode(file); + } + + csync_debug_ping(2); if ( recursive ) { - if ( !strcmp(file, "/") ) - asprintf(&where_rec, "or 1"); - else - asprintf(&where_rec, "UNION ALL SELECT filename from file where filename > '%s/' " - "and filename < '%s0'", - url_encode(file), url_encode(file)); + csync_debug_ping(2); + ftl=csdbGetFilesRecursiveFilename_(encoded_file); + csync_debug_ping(2); + } else { + csync_debug_ping(2); + ftl=csdbGetFilesFilename_(encoded_file); + csync_debug_ping(2); } - - SQL_BEGIN("Checking for removed files", - "SELECT filename from file where " - "filename = '%s' %s ORDER BY filename", url_encode(file), where_rec) - { - const char *filename = url_decode(SQL_V(0)); + csync_debug_ping(2); + for(t=ftl;t;t=t->next) { + const char *filename = url_decode(t->value2); if ( lstat_strict(prefixsubst(filename), &st) != 0 || csync_check_pure(filename) ) textlist_add(&tl, filename, 0); - } SQL_END; + } + textlist_free(ftl); + csync_debug_ping(2); for (t = tl; t != 0; t = t->next) { if (!init_run) csync_mark(t->value, 0, 0); - SQL("Removing file from DB. It isn't with us anymore.", - "DELETE FROM file WHERE filename = '%s'", - url_encode(t->value)); + csdbDeleteFileFilename_(url_encode(t->value)); } + csync_debug_ping(2); textlist_free(tl); + csync_debug_ping(2); - if ( recursive ) - free(where_rec); } int csync_check_mod(const char *file, int recursive, int ignnoent, int init_run) @@ -219,39 +218,36 @@ int csync_check_mod(const char *file, in switch ( check_type ) { - case 2: + case 2: { + struct textlist *tl, *t; csync_debug(2, "Checking %s.\n", file); checktxt = csync_genchecktxt(&st, file, 0); if (csync_compare_mode) printf("%s\n", file); - SQL_BEGIN("Checking File", - "SELECT checktxt FROM file WHERE " - "filename = '%s'", url_encode(file)) - { - if ( !csync_cmpchecktxt(checktxt, - url_decode(SQL_V(0))) ) { - csync_debug(2, "File has changed: %s\n", file); - this_is_dirty = 1; - } - } SQL_FIN { - if ( SQL_COUNT == 0 ) { - csync_debug(2, "New file: %s\n", file); - this_is_dirty = 1; + tl=csdbGetFilesFilename_(url_encode(file)); + if(tl) { + for(t=tl;t;t=t->next) { + if ( !csync_cmpchecktxt(checktxt, + url_decode(t->value)) ) { + csync_debug(2, "File has changed: %s\n", file); + this_is_dirty = 1; + } } - } SQL_END; + } else { + csync_debug(2, "New file: %s\n", file); + this_is_dirty = 1; + } if ( this_is_dirty && !csync_compare_mode ) { - SQL("Adding or updating file entry", - "INSERT INTO file (filename, checktxt) " - "VALUES ('%s', '%s')", - url_encode(file), url_encode(checktxt)); + csdbAddFileFilename_Check_(url_encode(file), url_encode(checktxt)); if (!init_run) csync_mark(file, 0, 0); } dirdump_this = 1; dirdump_parent = 1; /* fall thru */ + } case 1: if ( !recursive ) break; if ( !S_ISDIR(st.st_mode) ) break; diff -ruNp -X excludefiles trunk.409/conn.c localwork/conn.c --- trunk.409/conn.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/conn.c 2009-11-24 18:12:09.000000000 +0100 @@ -172,26 +172,20 @@ int conn_check_peer_cert(const char *pee { char certdata[peercert->size*2 + 1]; - + struct textlist *tl; for (i=0; isize; i++) sprintf(certdata+i*2, "%02X", peercert->data[i]); certdata[peercert->size*2] = 0; - - SQL_BEGIN("Checking peer x509 certificate.", - "SELECT certdata FROM x509_cert WHERE peername = '%s'", - url_encode(peername)) - { - if (!strcmp(SQL_V(0), certdata)) - cert_is_ok = 1; - else - cert_is_ok = 0; - } SQL_END; + tl=csdbGetCertPeername_(url_encode(peername)); + if (tl&&!strcmp(tl->value, certdata)) + cert_is_ok = 1; + else + cert_is_ok = 0; + if(tl) textlist_free(tl); if (cert_is_ok < 0) { csync_debug(1, "Adding peer x509 certificate to db: %s\n", certdata); - SQL("Adding peer x509 sha1 hash to database.", - "INSERT INTO x509_cert (peername, certdata) VALUES ('%s', '%s')", - url_encode(peername), url_encode(certdata)); + csdbAddCertPeername_Cert_(url_encode(peername), url_encode(certdata)); return 1; } diff -ruNp -X excludefiles trunk.409/csdb.c localwork/csdb.c --- trunk.409/csdb.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/csdb.c 2009-11-24 18:22:34.000000000 +0100 @@ -0,0 +1,191 @@ +#include "csync2.h" +#include "csdb.h" +#include "ipc.h" +#include +#include +#include +#include +#include +#include +#include +#include + +int csdbCheckDirtyFilename_(const char *filename) { + int c; + struct textlist *tl=NULL; + c=cdb_sqlselect("check if dirty exists","0",&tl,"SELECT 1 FROM dirty WHERE filename = '%s' LIMIT 1",filename); + textlist_free(tl); + return c; +} + +void csdbDeleteDirtyFilename_Peername_(const char *filename, const char * peername) { + cdb_sql("Remove dirty-file entry.", + "DELETE FROM dirty WHERE filename = '%s'" + "%s%s%s", filename,peername?" AND peername ='":"",peername?peername:"",peername?"'":""); +} +void csdbDeleteFileFilename_(const char *filename) { + cdb_sql("Remove old file from file db", + "DELETE FROM file WHERE filename = '%s'", filename); +} +void csdbAddFileFilename_Check_(const char *filename, const char *checktxt) { + cdb_sql( + "Add new file", + "INSERT INTO file (filename, checktxt) VALUES " + "('%s', '%s')", filename, checktxt); +} +void csdbAddDirtyFilename_Flags_Myname_Peername_(const char *filename, int flags, const char *myname, const char *peername) { + if(flags==DIRTY_FORCE) { + cdb_sql("Marking File forced Dirty", + "REPLACE INTO dirty (filename, force, myname, peername) " + "VALUES ('%s', 1, '%s', '%s')", + filename, myname, peername) ; + } else { + cdb_sql("Marking File Dirty", + "INSERT INTO dirty (filename, force, myname, peername) " + "VALUES ('%s', 0, '%s', '%s')", + filename, myname, peername) ; + } +} +struct textlist *csdbGetPeers(void) { + struct textlist *tl = 0; + cdb_sqlselect("Get hosts from dirty table","0",&tl,"SELECT peername FROM dirty GROUP BY peername ORDER BY random()"); + return tl; +} +struct textlist *csdbGetDirtyFilesPeername_Maxamount_(const char *peername, int limit) { + struct textlist *tl = 0; + char *limit_work=""; + if(limit) { + asprintf(&limit_work," LIMIT %d",limit); + } + if(peername) { + cdb_sqlselect("Dirty Files for peer","01b",&tl, + "SELECT filename, myname, force FROM dirty WHERE peername = '%s' " + "ORDER by filename ASC%s",peername,limit_work); + } else { + cdb_sqlselect("All Dirty Files","012b",&tl, + "SELECT filename, myname, peername, force FROM dirty " + "ORDER by filename ASC%s",limit_work); + } + if(limit) free(limit_work); + return tl; +} +struct textlist *csdbGetDirty(void) { + struct textlist *tl = 0; + cdb_sqlselect("DUMP dirty DB", "012b",&tl,"SELECT filename, myname, peername, force FROM dirty"); + return tl; +} +void csdbUpdateDirtyFilename_Recursive_Force_(char *name,int recursive, int force) { + char *sql; + if(recursive) { + if(name) { + asprintf(&sql, + "UPDATE dirty SET force = %d WHERE " + "filename = > '%s/' AND " + "filename < '%s0'", force, name, name); + } else { + asprintf(&sql,"UPDATE dirty SET force = %d",force); + } + cdb_sql("Mark subdirs as to be forced","%s",sql); + free(sql); + } + if(name) { + asprintf(&sql,"UPDATE dirty SET force = %d WHERE filename = '%s'", force,name); + cdb_sql("Mark entry as to be forced","%s",sql); + } +} +struct textlist *csdbGetFilesNamesonly(void) { + struct textlist *tl = 0; + cdb_sqlselect("Query file DB","0",&tl,"SELECT filename FROM file"); + return tl; +} +struct textlist *csdbGetFilesRecursiveFilename_(const char *filename) { + struct textlist *tl = 0; + char *where=""; + if(filename) { + asprintf(&where, + " WHERE filename= '%s' " + "UNION ALL SELECT checktxt, filename FROM file WHERE " + "filename > '%s/' " + "AND filename < '%s0'",filename,filename,filename); + } + cdb_sqlselect("Query file DB", "01", &tl,"SELECT checktxt, filename FROM file%s ORDER BY filename",where); + csync_debug_ping(2); + if(filename) free(where); + csync_debug_ping(2); + return tl; +} +struct textlist *csdbGetFilesFilename_(const char *filename) { + struct textlist *tl = 0; + char *where=""; + if(filename) { + asprintf(&where," WHERE filename= '%s'",filename); + } + cdb_sqlselect("Query file DB", "01", &tl,"SELECT checktxt, filename FROM file%s ORDER BY filename",where); + csync_debug_ping(2); + if(filename) free(where); + csync_debug_ping(2); + return tl; +} + +struct textlist *csdbGetHintsMaxamount_(int limit) { + struct textlist *tl = NULL; + char *limit_work=""; + if(limit) { + asprintf(&limit_work," LIMIT %d",limit); + } + cdb_sqlselect("Dump all hints", "0b", &tl, "SELECT filename, recursive FROM hint%s",limit_work); + if(limit) free(limit_work); + return tl; +} + +void csdbDeleteHintFilename_Flags_(const char *filename, int flags) { + cdb_sql("Remove processed hint.", "DELETE FROM hint WHERE filename = '%s' " "and recursive = %d", filename, flags); +} +void csdbAddHintFilename_Flags_(const char *filename, int flags) { + cdb_sql("Adding Hint", "INSERT INTO hint (filename, recursive) " "VALUES ('%s', %d)", filename, flags); +} + +void csdbAddActionFilename_Command_Logfile_(const char *filename, const char *command, const char *logfile) { + cdb_sql( + "Add new action", + "INSERT INTO action (filename, command, logfile) " + "VALUES ('%s', '%s', '%s')", + filename, command, logfile); +} +struct textlist *csdbGetActionCommand_Logfile_(const char *command, const char *logfile) { + struct textlist *tl = NULL; + cdb_sqlselect( + "Get action", "0", &tl, + "SELECT filename FROM action WHERE command = '%s' AND logfile = '%s'", + command,logfile); + return tl; +} +void csdbDeleteActionFilename_Command_Logfile_(const char *filename, const char *command, const char *logfile) { + cdb_sql( + "Add new action", + "DELETE FROM action WHERE " + "command = '%s' AND " + "logfile = '%s' AND " + "filename = '%s'", + command,logfile,filename); +} +struct textlist *csdbGetActionCommands(void) { + struct textlist *tl = NULL; + cdb_sqlselect( + "Get actions", "01", &tl, + "SELECT command, logfile FROM action GROUP BY command, logfile"); + return tl; +} +struct textlist *csdbGetCertPeername_(const char *peername) { + struct textlist *tl = NULL; + cdb_sqlselect( + "Get certs", "0", &tl, + "SELECT certdata FROM x509_cert WHERE peername = '%s'",peername); + return tl; +} +void csdbAddCertPeername_Cert_(const char *peername, const char *certdata) { + cdb_sql( + "Add new file", + "INSERT INTO x509_cert (peername, certdata) VALUES " + "('%s', '%s')", peername, certdata); +} diff -ruNp -X excludefiles trunk.409/csdbd.c localwork/csdbd.c --- trunk.409/csdbd.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/csdbd.c 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,252 @@ +#include "csync2.h" +#include "ipc.h" +#include "csdb.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int csdbh=-1; + + +int cdb_simplesend(int handle, int message, char *data) { + struct csdbmsg *msg; + int s,ms; + s=strlen(data)+1; + ms=sizeof(msg[0])+s; + msg=malloc(ms); + if(!msg) { + csync_debug(0,"out of memory"); + return -1; + } + msg[0].message=message; + msg[0].size=s; + strcpy(msg[0].data,data); + ipc_sndmsg(handle,msg,ms); + csync_debug(3,"simplesend:%d, %d, %d, %d (%s)\n",sizeof(int),ms, msg[0].message,msg[0].size,msg[0].data); + free(msg); + return 0; +} + +void cdb_sql(const char *err, const char *fmt, ...) { + char *sql; + va_list ap; + va_start(ap, fmt); + vasprintf(&sql, fmt, ap); + va_end(ap); + if(csdbh>=0) { + struct csdbmsg_sql_status sr; + int r; +// cdb_simplesend(csdbh,CDB_PING,err); +// r=ipc_rcvmsga(csdbh,&sr,sizeof(sr)); +// if(r!=sr||sr.message!=RDB_SQLSTATUS) { +// csync_debug(0,"protocol error"); +// } + cdb_simplesend(csdbh,CDB_SQLPLAIN,sql); + r=ipc_rcvmsg(csdbh,&sr,sizeof(sr)); + if(r!=sizeof(sr) || sr.message!=RDB_SQLSTATUS) { + csync_debug(0,"protocol error, received %d bytes",r); + } + } else { + csync_debug(2, "SQL: %s\n", sql); + csync_db_sql(err,sql); + } + free(sql); +} + +int cdb_sqlselect(const char *err, const char *returnformat, struct textlist **ptl,const char *fmt, ...) { + int count=0; + int r=-1; + int i; + int cc=strlen(returnformat); + char *sql; + char *total; + char *args[CSDBMAXCOLUMNS]; + int b=-1; + int bv; + int maxa; + va_list ap; + va_start(ap, fmt); + vasprintf(&sql, fmt, ap); + va_end(ap); + asprintf(&total,"%c%s",cc+'0',sql); + for(i=0;i=0) { + cdb_simplesend(csdbh,CDB_SQLSELECT,total); + while(1) { + void *msgb=NULL; + struct csdbmsg *msg; + struct csdbmsg_sql_row *row; + int buffer_size,n; + n=ipc_rcvmsga(csdbh,(void *)&msgb,&buffer_size); + if(n<0) { + csync_debug(0,"Connection closed unexptectedly"); + close(csdbh); + csdbh=-1; + if(msgb) free(msgb); + break; + } + if(buffer_size='0' && returnformat[i]<='9') { + n=(int)(returnformat[i]-'0'); + args[n]=&(row[0].data[row[0].coloff[i]]); + if(maxa=0) bv=atoi(&(row[0].data[row[0].coloff[b]])); + switch(maxa) { + case 0: textlist_addorder(ptl, args[0], bv); break; + case 1: textlist_addorder2(ptl, args[0], args[1], bv); break; + case 2: textlist_addorder3(ptl, args[0], args[1], args[2], bv); break; + } + if(msgb) free(msgb); + } + } + if(sql) free(sql); + if(total) free(total); + return count; +} + + +int csdb_connect() { + if(!csync_csdbsocket) { + csync_debug(0,"No socket defined"); + return -1; + } + csdbh=ipc_connect(csync_csdbsocket); + if(csdbh<0) return csdbh; + return 0; +} +int cdb_sqlstatus(int c, int rows, int result) { + struct csdbmsg_sql_status sr; + sr.message=RDB_SQLSTATUS; + sr.size=sizeof(sr.rows)+sizeof(sr.result); + sr.rows=rows; + sr.result=result; + ipc_sndmsg(c,&sr,sizeof(sr)); +} +int cdbd_sql(ipch hserver,int c,struct csdbmsg *msg, int buffer_size) { + csync_debug(2, "CDB_SQLPLAIN: %s;\n", msg[0].data); + SQL(msg[0].data,"%s",msg[0].data); + csync_debug(2, "CDB_SQLPLAIN(%d) finished\n",c, msg[0].data); + cdb_sqlstatus(c,0,0); + return 0; +} + +int cdbd_sqlselect(ipch hserver,int c,struct csdbmsg *msg, int buffer_size) { + int cc; + cc=(int) (msg[0].data[0]-'0'); + csync_debug(2, "CDB_SQLSELECT(%d): %d, %s;\n",c,cc,&(msg[0].data[1])); + SQL_BEGIN(msg[0].data,"%s",&(msg[0].data[1])) { + struct csdbmsg_sql_row *row; + int ps,i,off; + ps=sizeof(*row); + for(i=0;i0) { + int c; + while((c=is_ipc_in_fds(hserver,&rfds,1))>=0) handle_connection(hserver,c); + } + } + ipc_destroy(hserver); + return 0; +} + diff -ruNp -X excludefiles trunk.409/csdb.h localwork/csdb.h --- trunk.409/csdb.h 1970-01-01 01:00:00.000000000 +0100 +++ localwork/csdb.h 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,41 @@ +#ifndef CSDB_H +#define CSDB_H 1 +/* csdb internal stuff */ +enum csdb_messages { + CDB_PING, + CDB_SQLPLAIN, + CDB_SQLSELECT, + CDB_SQLPREPARE, + CDB_SQLNEXT, + CDB_SQLCLOSE, + + RDB_PONG, + RDB_SQLSTATUS, + RDB_SQLROW +}; + +struct csdbmsg { + int message; + int size; + char data[]; +}; +struct csdbmsg_sql_status { + int message; + int size; //(sizeof rows,result) + int rows; + int result; +}; +#define CSDBMAXCOLUMNS 5 +struct csdbmsg_sql_row { + int message; + int size; //(sizeof rows,result) + int columns; + int colsize[CSDBMAXCOLUMNS]; + int coloff[CSDBMAXCOLUMNS]; + char data[]; +}; +#define CDBSQLPLAIN(e, s, ...) cdb_sql(e, s, ##__VA_ARGS__) +extern void cdb_sql(const char *err, const char *fmt, ...); +extern int cdb_sqlselect(const char *err, const char *returnformat, struct textlist **ptl,const char *fmt, ...); + +#endif /* CSDB_H */ diff -ruNp -X excludefiles trunk.409/csync2.c localwork/csync2.c --- trunk.409/csync2.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/csync2.c 2009-11-24 23:58:57.000000000 +0100 @@ -62,6 +62,7 @@ int csync_server_child_pid = 0; int csync_timestamps = 0; int csync_new_force = 0; int csync_port = 30865; +int csync_limit_work = 0; int csync_dump_dir_fd = -1; @@ -83,7 +84,8 @@ enum { MODE_LIST_DIRTY, MODE_REMOVE_OLD, MODE_COMPARE, - MODE_SIMPLE + MODE_SIMPLE, + MODE_DB_D }; void help(char *cmd) @@ -171,6 +173,9 @@ PACKAGE_STRING " - cluster synchronizati "\n" " -t Print timestamps to debug output (e.g. for profiling).\n" "\n" +" -l count\n" +" Limit work to count database entries.\n" +"\n" " -s filename\n" " Print timestamps also to this file.\n" "\n" @@ -235,9 +240,7 @@ static int csync_server_loop(int single_ if ( bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)) < 0 ) goto error; if ( listen(listenfd, 5) < 0 ) goto error; - /* we want to "cleanly" shutdown if the connection is lost unexpectedly */ signal(SIGPIPE, SIG_IGN); - /* server is not interested in its childs, prevent zombies */ signal(SIGCHLD, SIG_IGN); printf("Csync2 daemon running. Waiting for connections.\n"); @@ -250,9 +253,6 @@ static int csync_server_loop(int single_ fflush(stdout); fflush(stderr); if (single_connect || !fork()) { - /* need to restore default SIGCHLD handler in the session, - * as we may need to wait on them in action.c */ - signal(SIGCHLD, SIG_DFL); csync_server_child_pid = getpid(); fprintf(stderr, "<%d> New connection from %s:%u.\n", csync_server_child_pid, @@ -298,7 +298,7 @@ int main(int argc, char ** argv) return 1; } - while ( (opt = getopt(argc, argv, "W:s:Ftp:G:P:C:D:N:HBAIXULSTMRvhcuoimfxrd")) != -1 ) { + while ( (opt = getopt(argc, argv, "l:W:s:Ftp:G:P:C:D:N:HBAIXULSTMRvhcuoimfxrdb")) != -1 ) { switch (opt) { case 'W': csync_dump_dir_fd = atoi(optarg); @@ -321,6 +321,9 @@ int main(int argc, char ** argv) case 'p': csync_port = atoi(optarg); break; + case 'l': + csync_limit_work = atoi(optarg); + break; case 'G': active_grouplist = optarg; break; @@ -397,6 +400,10 @@ int main(int argc, char ** argv) if ( mode != MODE_NONE ) help(argv[0]); mode = MODE_FORCE; break; + case 'b': + if ( mode != MODE_NONE ) help(argv[0]); + mode = MODE_DB_D; + break; case 'H': if ( mode != MODE_NONE ) help(argv[0]); mode = MODE_LIST_HINT; @@ -543,7 +550,7 @@ int main(int argc, char ** argv) file_config, strerror(errno)); yyparse(); fclose(yyin); - +// cslogopen("csync2"); { const struct csync_group *g; for (g=csync_group; g; g=g->next) @@ -552,7 +559,16 @@ int main(int argc, char ** argv) found_a_group:; } - csync_db_open(file_database); + if(mode!=MODE_DB_D&&csync_csdbsocket) { + int r; + r=csdb_connect(); + if(r) { + fprintf(stderr,"failed to connect to csdbd\n"); + exit(-1); + } + } else { + csync_db_open(file_database); + } for (i=optind; i < argc; i++) on_cygwin_lowercase(argv[i]); @@ -590,19 +606,10 @@ found_a_group:; case MODE_CHECK_AND_UPDATE: if ( argc == optind ) { - SQL_BEGIN("Check all hints", - "SELECT filename, recursive FROM hint") - { - textlist_add(&tl, url_decode(SQL_V(0)), - atoi(SQL_V(1))); - } SQL_END; - + tl= csdbGetHintsMaxamount_(csync_limit_work); for (t = tl; t != 0; t = t->next) { csync_check(t->value, t->intvalue, init_run); - SQL("Remove processed hint.", - "DELETE FROM hint WHERE filename = '%s' " - "and recursive = %d", - url_encode(t->value), t->intvalue); + csdbDeleteHintFilename_Flags_(url_encode(t->value), t->intvalue); } textlist_free(tl); @@ -649,6 +656,9 @@ found_a_group:; conn_printf("OK (cmd_finished).\n"); csync_daemon_session(); break; + case MODE_DB_D: + csdb_daemon(); + break; case MODE_MARK: for (i=optind; i < argc; i++) { @@ -659,23 +669,20 @@ found_a_group:; csync_mark(pfname, 0, 0); if ( recursive ) { - char *where_rec = ""; - - if ( !strcmp(realname, "/") ) - asprintf(&where_rec, "or 1"); - else - asprintf(&where_rec, "UNION ALL SELECT filename from file where filename > '%s/' " - "and filename < '%s0'", - url_encode(pfname), url_encode(pfname)); - - SQL_BEGIN("Adding dirty entries recursively", - "SELECT filename FROM file WHERE filename = '%s' %s", - url_encode(pfname), where_rec) - { - char *filename = strdup(url_decode(SQL_V(0))); + struct textlist *tl,*t; + if(0==strcmp(realname,"/")) { + /* Mark all */ + tl=csdbGetFilesRecursiveFilename_(NULL); + } else { + /* "just" sub select */ + tl=csdbGetFilesRecursiveFilename_(url_encode(pfname)); + } + for (t = tl; t != 0; t = t->next) { + char *filename = strdup(url_decode(t->value2)); csync_mark(filename, 0, 0); free(filename); - } SQL_END; + } + textlist_free(tl); } free(pfname); } @@ -684,60 +691,52 @@ found_a_group:; case MODE_FORCE: for (i=optind; i < argc; i++) { char *realname = getrealfn(argv[i]); - char *where_rec = ""; - - if ( recursive ) { - if ( !strcmp(realname, "/") ) - asprintf(&where_rec, "or 1"); - else - asprintf(&where_rec, "or (filename > '%s/' " - "and filename < '%s0')", - url_encode(realname), url_encode(realname)); + char *encodedname = NULL; + if(strcmp(realname,"/")) { + encodedname=strdup(url_encode(realname)); } - - SQL("Mark file as to be forced", - "UPDATE dirty SET force = 1 WHERE filename = '%s' %s", - url_encode(realname), where_rec); - - if ( recursive ) - free(where_rec); + csdbUpdateDirtyFilename_Recursive_Force_(encodedname,recursive,1); + if(encodedname) free(encodedname); } break; - case MODE_LIST_HINT: + case MODE_LIST_HINT: { retval = 2; - SQL_BEGIN("DB Dump - Hint", - "SELECT recursive, filename FROM hint ORDER BY filename") - { - printf("%s\t%s\n", SQL_V(0), url_decode(SQL_V(1))); + struct textlist *tl,*t; + tl=csdbGetHintsMaxamount_(0); + for(t=tl;t;t=t->next) { + printf("%d\t%s\n", t->intvalue, url_decode(t->value)); retval = -1; - } SQL_END; + } + textlist_free(tl); break; - - case MODE_LIST_FILE: + } + case MODE_LIST_FILE: { retval = 2; - SQL_BEGIN("DB Dump - File", - "SELECT checktxt, filename FROM file ORDER BY filename") - { - if (csync_find_next(0, url_decode(SQL_V(1)))) { - printf("%s\t%s\n", url_decode(SQL_V(0)), url_decode(SQL_V(1))); + struct textlist *tl,*t; + tl=csdbGetFilesFilename_(NULL); + for(t=tl;t;t=t->next) { + if (csync_find_next(0, url_decode(t->value2))) { + printf("%s\t%s\n", url_decode(t->value), url_decode(t->value2)); retval = -1; } - } SQL_END; + } + textlist_free(tl); break; - - case MODE_LIST_SYNC: + } + case MODE_LIST_SYNC: { retval = 2; - SQL_BEGIN("DB Dump - File", - "SELECT checktxt, filename FROM file ORDER BY filename") - { - if ( csync_match_file_host(url_decode(SQL_V(1)), argv[optind], argv[optind+1], 0) ) { - printf("%s\t%s\n", url_decode(SQL_V(0)), url_decode(SQL_V(1))); + struct textlist *tl, *t; + tl=csdbGetFilesFilename_(NULL); + for(t=tl;t;t=t->next) { + if ( csync_match_file_host(url_decode(t->value2), argv[optind], argv[optind+1], 0) ) { + printf("%s\t%s\n", url_decode(t->value), url_decode(t->value2)); retval = -1; } - } SQL_END; + } + textlist_free(tl); break; - + } case MODE_TEST_SYNC: { char *realname; if (init_run && init_run_with_removals) @@ -778,19 +777,19 @@ found_a_group:; break; } - case MODE_LIST_DIRTY: + case MODE_LIST_DIRTY: { retval = 2; - SQL_BEGIN("DB Dump - Dirty", - "SELECT force, myname, peername, filename FROM dirty ORDER BY filename") - { - if (csync_find_next(0, url_decode(SQL_V(3)))) { - printf("%s\t%s\t%s\t%s\n", atoi(SQL_V(0)) ? "force" : "chary", - url_decode(SQL_V(1)), url_decode(SQL_V(2)), url_decode(SQL_V(3))); + struct textlist *tl,*t; + tl=csdbGetDirtyFilesPeername_Maxamount_(NULL,0); + for(t=tl;t;t=t->next) { + if (csync_find_next(0, url_decode(t->value))) { + printf("%s\t%s\t%s\t%s\n", t->intvalue ? "force" : "chary", + url_decode(t->value2), url_decode(t->value3), url_decode(t->value)); retval = -1; } - } SQL_END; + } break; - + } case MODE_REMOVE_OLD: if ( active_grouplist ) csync_fatal("Never run -R with -G!\n"); diff -ruNp -X excludefiles trunk.409/csync2.cfg localwork/csync2.cfg --- trunk.409/csync2.cfg 2009-11-24 18:04:53.000000000 +0100 +++ localwork/csync2.cfg 2009-11-25 14:33:32.000000000 +0100 @@ -56,3 +56,22 @@ # on host[12]: /export/users; # on *: /home; # } + +# If used in a secure environment you don't have to use ssl: +# nossl * *; + +# To prevent lock contention on the database on a heavily updated system, use a +# metadata (database) daemon. +# csdb-socket /var/lib/csync2/csdb-connection; +# You need to start the daemon with csync2 -bBA before using this. + + +# When transferring large files, or be able to move (future, not now) +# transferred files, change the temp dir where csync2 receives and constructs +# the files. +# tempdir /my/data/partition/temp; + +# When having loads of files, it might be worth it to split the databases +# You can then vacuum the files seperately +# split-databases; +# The database will be split per table diff -ruNp -X excludefiles trunk.409/csync2.h localwork/csync2.h --- trunk.409/csync2.h 2009-11-24 18:04:53.000000000 +0100 +++ localwork/csync2.h 2009-11-25 14:24:30.000000000 +0100 @@ -82,7 +82,6 @@ extern int conn_gets(char *s, int size); /* db.c */ - extern void csync_db_open(const char *file); extern void csync_db_close(); @@ -141,6 +140,7 @@ extern void * csync_db_colblob(void *stm extern int db_blocking_mode; extern int db_sync_mode; +extern int csync_db_split; /* rsync.c */ @@ -205,9 +205,11 @@ struct textlist; struct textlist { struct textlist *next; + struct textlist *tail; int intvalue; char *value; char *value2; + char *value3; }; static inline void textlist_add(struct textlist **listhandle, const char *item, int intitem) @@ -217,7 +219,13 @@ static inline void textlist_add(struct t (*listhandle)->intvalue = intitem; (*listhandle)->value = strdup(item); (*listhandle)->value2 = 0; + (*listhandle)->value3 = 0; (*listhandle)->next = tmp; + if(tmp) { + (*listhandle)->tail=(tmp)->tail; + } else { + (*listhandle)->tail=*listhandle; + } } static inline void textlist_add2(struct textlist **listhandle, const char *item, const char *item2, int intitem) @@ -227,7 +235,86 @@ static inline void textlist_add2(struct (*listhandle)->intvalue = intitem; (*listhandle)->value = strdup(item); (*listhandle)->value2 = strdup(item2); + (*listhandle)->value3 = 0; + (*listhandle)->next = tmp; + if(tmp) { + (*listhandle)->tail=(tmp)->tail; + } else { + (*listhandle)->tail=*listhandle; + } +} + +static inline void textlist_add3(struct textlist **listhandle, const char *item, const char *item2, const char *item3,int intitem) +{ + struct textlist *tmp = *listhandle; + *listhandle = malloc(sizeof(struct textlist)); + (*listhandle)->intvalue = intitem; + (*listhandle)->value = strdup(item); + (*listhandle)->value2 = strdup(item2); + (*listhandle)->value3 = strdup(item3); (*listhandle)->next = tmp; + if(tmp) { + (*listhandle)->tail=(tmp)->tail; + } else { + (*listhandle)->tail=*listhandle; + } +} + +static inline void textlist_addorder(struct textlist **listhandle, const char *item, int intitem) +{ + struct textlist *tmp = *listhandle; + struct textlist *new; + new = malloc(sizeof(struct textlist)); + new->intvalue = intitem; + new->value = strdup(item); + new->value2 = 0; + new->value3 = 0; + new->next = NULL; + if(tmp) { + (tmp)->tail->next=new; + (tmp)->tail=new; + } else { + new->tail=new; + *listhandle=new; + } +} + +static inline void textlist_addorder2(struct textlist **listhandle, const char *item, const char *item2, int intitem) +{ + struct textlist *tmp = *listhandle; + struct textlist *new; + new = malloc(sizeof(struct textlist)); + new->intvalue = intitem; + new->value = strdup(item); + new->value2 = strdup(item2); + new->value3 = 0; + new->next = NULL; + if(tmp) { + (tmp)->tail->next=new; + (tmp)->tail=new; + } else { + new->tail=new; + *listhandle=new; + } +} + +static inline void textlist_addorder3(struct textlist **listhandle, const char *item, const char *item2, const char *item3,int intitem) +{ + struct textlist *tmp = *listhandle; + struct textlist *new; + new = malloc(sizeof(struct textlist)); + new->intvalue = intitem; + new->value = strdup(item); + new->value2 = strdup(item2); + new->value3 = strdup(item3); + new->next = NULL; + if(tmp) { + (tmp)->tail->next=new; + (tmp)->tail=new; + } else { + new->tail=new; + *listhandle=new; + } } static inline void textlist_free(struct textlist *listhandle) @@ -236,6 +323,8 @@ static inline void textlist_free(struct while (listhandle != 0) { next = listhandle->next; free(listhandle->value); + if ( listhandle->value3 ) + free(listhandle->value3); if ( listhandle->value2 ) free(listhandle->value2); free(listhandle); @@ -329,6 +418,9 @@ extern struct csync_group *csync_group; extern struct csync_prefix *csync_prefix; extern struct csync_nossl *csync_nossl; +extern char *csync_tempdir; +extern char *csync_csdbsocket; + extern int csync_error_count; extern int csync_debug_level; extern FILE *csync_debug_out; @@ -342,6 +434,8 @@ extern int csync_timestamps; extern int csync_new_force; extern int csync_port; +extern int csync_limit_work; + extern char myhostname[]; extern char *active_grouplist; extern char *active_peerlist; @@ -388,5 +482,41 @@ static inline char *on_cygwin_lowercase( return s; } +/* csdb.c */ +extern void csdbDeleteDirtyFilename_Peername_(const char *filename, const char * peername); +extern void csdbAddHintFilename_Flags_(const char *filename, int flags); +extern void csdbAddDirtyFilename_Flags_Myname_Peername_(const char *filename, int flags, const char *myname, const char *peername); +enum dirty_flags { DIRTY_NONE=0, DIRTY_FORCE =1 }; +extern struct textlist *csdbGetDirtyFilesPeername_Maxamount_(const char *peername, int limit); +extern struct textlist *csdbGetPeers(void); +extern void csdbDeleteFileFilename_(const char *filename); +extern struct textlist *csdbGetFilesNamesonly(void); +extern struct textlist *csdbGetFilesFilename_(const char *filename); +extern struct textlist *csdbGetFilesRecursiveFilename_(const char *filename); +extern struct textlist *csdbGetDirty(void); +extern struct textlist *csdbGetHintsMaxamount_(int limit); +extern void csdbDeleteHintFilename_Flags_(const char *filename, int flags); +extern void csdbUpdateDirtyFilename_Recursive_Force_(char *name,int recursive, int force); +extern int csdbCheckDirtyFilename_(const char *filename); +extern void csdbAddFileFilename_Check_(const char *filename, const char *checktxt); +extern void csdbAddActionFilename_Command_Logfile_(const char *filename, const char *command, const char *logfile); +extern struct textlist *csdbGetActionCommand_Logfile_(const char *command, const char *logfile); +extern struct textlist *csdbGetActionCommands(void); +extern void csdbAddCertPeername_Cert_(const char *peername, const char *certdata); +extern struct textlist *csdbGetCertPeername_(const char *peername); + +/* csdbd.c */ +extern int csdb_daemon(void); +extern int csdb_connect(void); + +/* logger.c */ +extern int cslogopen(char *name); +extern void cslog(int prio, char *format, ...); +enum CSLOGSEVERITY { + CS_DEBUG, + CS_INFO, + CS_WARN, + CS_ERROR +}; #endif /* CSYNC2_H */ diff -ruNp -X excludefiles trunk.409/daemon.c localwork/daemon.c --- trunk.409/daemon.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/daemon.c 2009-11-24 18:12:09.000000000 +0100 @@ -54,15 +54,14 @@ int csync_unlink(const char *filename, i int csync_check_dirty(const char *filename, const char *peername, int isflush) { int rc = 0; + int count=0; csync_check(filename, 0, 0); if (isflush) return 0; - SQL_BEGIN("Check if file is dirty", - "SELECT 1 FROM dirty WHERE filename = '%s' LIMIT 1", - url_encode(filename)) - { + count=csdbCheckDirtyFilename_(url_encode(filename)); + if(count>0) { rc = 1; cmd_error = "File is also marked dirty here!"; - } SQL_END; + } if (rc && peername) csync_mark(filename, peername, 0); return rc; @@ -71,27 +70,18 @@ int csync_check_dirty(const char *filena void csync_file_update(const char *filename, const char *peername) { struct stat st; - SQL("Removing file from dirty db", - "delete from dirty where filename = '%s' and peername = '%s'", - url_encode(filename), peername); + csdbDeleteDirtyFilename_Peername_(url_encode(filename),peername); if ( lstat_strict(prefixsubst(filename), &st) != 0 || csync_check_pure(filename) ) { - SQL("Removing file from file db", - "delete from file where filename = '%s'", - url_encode(filename)); + csdbDeleteFileFilename_(url_encode(filename)); } else { const char *checktxt = csync_genchecktxt(&st, filename, 0); - SQL("Insert record to file db", - "insert into file (filename, checktxt) values " - "('%s', '%s')", url_encode(filename), - url_encode(checktxt)); + csdbAddFileFilename_Check_(url_encode(filename), url_encode(checktxt)); } } void csync_file_flush(const char *filename) { - SQL("Removing file from dirty db", - "delete from dirty where filename ='%s'", - url_encode(filename)); + csdbDeleteDirtyFilename_Peername_(url_encode(filename),NULL); } int csync_file_backup(const char *filename) @@ -347,9 +337,7 @@ void csync_daemon_session() } break; case A_FLUSH: - SQL("Flushing dirty entry (if any) for file", - "DELETE FROM dirty WHERE filename = '%s'", - url_encode(tag[2])); + csdbDeleteDirtyFilename_Peername_(url_encode(tag[2]),NULL); break; case A_DEL: if (!csync_file_backup(tag[2])) @@ -436,18 +424,20 @@ void csync_daemon_session() cmd_error = strerror(errno); } break; - case A_LIST: - SQL_BEGIN("DB Dump - Files for sync pair", - "SELECT checktxt, filename FROM file %s%s%s ORDER BY filename", - strcmp(tag[2], "-") ? "WHERE filename = '" : "", - strcmp(tag[2], "-") ? url_encode(tag[2]) : "", - strcmp(tag[2], "-") ? "'" : "") - { - if ( csync_match_file_host(url_decode(SQL_V(1)), tag[1], peer, (const char **)&tag[3]) ) - conn_printf("%s\t%s\n", SQL_V(0), SQL_V(1)); - } SQL_END; + case A_LIST: { + struct textlist *tl,*t; + char *fn=NULL; + if(strcmp(tag[2],"-")) { + fn=url_encode(tag[2]); + } + tl=csdbGetFilesFilename_(fn); + for(t=tl;t;t=t->next) { + if ( csync_match_file_host(url_decode(t->value2), tag[1], peer, (const char **)&tag[3]) ) + conn_printf("%s\t%s\n", t->value, t->value2); + + } break; - + } case A_DEBUG: csync_debug_out = stdout; if ( tag[1][0] ) diff -ruNp -X excludefiles trunk.409/db.c localwork/db.c --- trunk.409/db.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/db.c 2009-11-25 14:27:40.000000000 +0100 @@ -118,8 +118,10 @@ void csync_db_maycommit() if ((now - last_wait_cycle) > 10) { SQL("COMMIT TRANSACTION", "COMMIT TRANSACTION"); - csync_debug(2, "Waiting 2 secs so others can lock the database (%d - %d)...\n", (int)now, (int)last_wait_cycle); - sleep(2); + if(!csync_csdbsocket) { + csync_debug(2, "Waiting 2 secs so others can lock the database (%d - %d)...\n", (int)now, (int)last_wait_cycle); + sleep(2); + } last_wait_cycle = 0; tqueries_counter = -10; begin_commit_recursion--; @@ -140,11 +142,37 @@ void csync_db_maycommit() return; } +void csync_create_db(const char *file, const char *query) { + IFESQL3(sqlite3,sqlite) *mydb = 0; + IFSQL3(int r;) + struct stat buf; + if(!csync_db_split) { + SQL_EXEC(db,query,0,0,0); + } else if(stat(file,&buf)==-1) { + IFESQL3(r=sqlite3_open(file, &mydb);, mydb = sqlite_open(file, 0, 0);) + if(mydb) { + in_sql_query++; + SQL_EXEC(mydb,query,0,0,0); + IFESQL3(sqlite3_close,sqlite_close)(mydb); + in_sql_query--; + } else { + csync_fatal("Can't open database: %s\n", file); + } + } +} +void csync_attach_db(const char *file, const char *name) { + char *query; + asprintf(&query,"ATTACH DATABASE \"%s\" as %s",file,name); + SQL_EXEC(db,query,0,0,0); +} void csync_db_open(const char *file) { + char *dbname; IFSQL3(int r;) - IFESQL3(r=sqlite3_open(file, &db);, - db = sqlite_open(file, 0, 0);) + dbname=file; + if(csync_db_split) asprintf(&dbname,"%s%s",file,"-file"); + IFESQL3(r=sqlite3_open(dbname, &db);, + db = sqlite_open(dbname, 0, 0);) if ( db == 0 IFSQL3(||r)) csync_fatal("Can't open database: %s\n", file); @@ -156,30 +184,52 @@ void csync_db_open(const char *file) " UNIQUE ( filename ) ON CONFLICT REPLACE" ")", 0, 0, 0); - SQL_EXEC(db, + if (!db_sync_mode) + SQL_EXEC(db, "PRAGMA synchronous = OFF", 0, 0, 0); + if(csync_db_split) { + free(dbname); + asprintf(&dbname,"%s%s",file,"-dirty"); + } + csync_create_db(dbname, "CREATE TABLE dirty (" " filename, force, myname, peername," " UNIQUE ( filename, peername ) ON CONFLICT IGNORE" - ")", - 0, 0, 0); - SQL_EXEC(db, + ")" + ); + if(csync_db_split) { + csync_attach_db(dbname,"dirty"); + free(dbname); + asprintf(&dbname,"%s%s",file,"-hint"); + } + csync_create_db(dbname, "CREATE TABLE hint (" " filename, recursive," " UNIQUE ( filename, recursive ) ON CONFLICT IGNORE" - ")", - 0, 0, 0); - SQL_EXEC(db, + ")"); + if(csync_db_split) { + csync_attach_db(dbname,"hint"); + free(dbname); + asprintf(&dbname,"%s%s",file,"-action"); + } + csync_create_db(dbname, "CREATE TABLE action (" " filename, command, logfile," " UNIQUE ( filename, command ) ON CONFLICT IGNORE" - ")", - 0, 0, 0); - SQL_EXEC(db, + ")"); + if(csync_db_split) { + csync_attach_db(dbname,"action"); + free(dbname); + asprintf(&dbname,"%s%s",file,"-x509cert"); + } + csync_create_db(dbname, "CREATE TABLE x509_cert (" " peername, certdata," " UNIQUE ( peername ) ON CONFLICT IGNORE" - ")", - 0, 0, 0); + ")"); + if(csync_db_split) { + csync_attach_db(dbname,"x509cert"); + free(dbname); + } if (!db_sync_mode) SQL_EXEC(db, "PRAGMA synchronous = OFF", 0, 0, 0); in_sql_query--; @@ -217,13 +267,27 @@ void csync_db_sql(const char *err, const while (1) { rc = SQL_EXEC(db, sql, 0, 0, 0); if ( rc != SQLITE_BUSY ) break; - if (busyc++ > get_dblock_timeout()) { db = 0; csync_fatal(DEADLOCK_MESSAGE); } + if (busyc++ > get_dblock_timeout()) { + if(!csync_csdbsocket) { + db = 0; + csync_fatal(DEADLOCK_MESSAGE); + } else { + csync_debug(0,DEADLOCK_MESSAGE); + csync_debug(0,"Continueing anyway"); + } + } csync_debug(2, "Database is busy, sleeping a sec.\n"); sleep(1); } - if ( rc != SQLITE_OK && err ) - csync_fatal("Database Error: %s [%d]: %s\n", err, rc, sql); + if ( rc != SQLITE_OK && err ) { + if(!csync_csdbsocket) { + csync_fatal("Database Error: %s [%d]: %s\n", err, rc, sql); + } else { + csync_debug(0,"Database Error: %s [%d]: %s\n", err, rc, sql); + /* We should return an error, so csdbd has a chance to close the connection */ + } + } free(sql); csync_db_maycommit(); diff -ruNp -X excludefiles trunk.409/error.c localwork/error.c --- trunk.409/error.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/error.c 2009-11-24 18:12:09.000000000 +0100 @@ -130,6 +130,7 @@ void csync_debug(int lv, const char *fmt va_start(ap, fmt); vfprintf(csync_debug_out, fmt, ap); va_end(ap); + fprintf(csync_debug_out,"\n"); csync_messages_printed++; } diff -ruNp -X excludefiles trunk.409/ipc.c localwork/ipc.c --- trunk.409/ipc.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/ipc.c 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,217 @@ +#include +#include +/* bug in sys/un.h */ +#define UNIX_PATH_MAX 108 +#include +#include +#include +#include +#include +#include +#include + +#include "ipc.h" + +#define IPCMAGIC 0xbabecafe + +int ipc_check_closed(int socket) { + int r; + struct pollfd test; + test.fd=socket; + test.events=POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL; + r=poll(&test,1,0); + printf("revents=%08x\n",test.revents); + return 0; +} + +ipch ipc_setup_listen(char *name) { + int i,s,r; + ipch h; + struct sockaddr_un *socknaam; + h=calloc(1,sizeof(*h)); + for(i=0;iconnection[i]=-1; + } + socknaam=malloc(sizeof(*socknaam)); + unlink(name); + if(!socknaam) { + free(h); + return NULL; + } + s=socket(PF_UNIX,SOCK_STREAM,0); + if(s<0) { + free(h); + return NULL; + } + h->listen_fd=s; + socknaam->sun_family=AF_UNIX; + strncpy(socknaam->sun_path,name,UNIX_PATH_MAX); + r=bind(s,(struct sockaddr *)socknaam,sizeof(*socknaam)); + if(r) { + close(s); + free(h); + return NULL; + } + r=listen(s,5); + if(r) { + close(s); + free(h); + return NULL; + } + return h; +} + +int ipc_destroy(ipch h) { + int i; + if(h->listen_fd>=0) close(h->listen_fd); + for(i=0;iconnection[i]>=0) close(h->connection[i]); + } + free(h); + return 0; +} + +int is_ipc_connection(ipch h, int fd) { + int i; + for(i=0;iconnection[i]==fd) return i; + } + return -1; +} + +int ipc_set_fds(ipch h,fd_set *fd,int autoaccept) { + int i,maxfd; + if(autoaccept) { + FD_SET(h->listen_fd,fd); + maxfd=h->listen_fd; + } else { + maxfd=-1; + } + for(i=0;iconnection[i]>maxfd) maxfd=h->connection[i]; + if(h->connection[i]>=0) FD_SET(h->connection[i],fd); + } + return maxfd; +} + +int is_ipc_in_fds(ipch h,fd_set *fd,int autoaccept) { + int i; + if(autoaccept && FD_ISSET(h->listen_fd,fd)) { + ipc_accept(h); + FD_CLR(h->listen_fd,fd); + } + for(i=0;iconnection[i]>=0 && FD_ISSET(h->connection[i],fd)) { + FD_CLR(h->connection[i],fd); + return h->connection[i]; + } + } + return -1; +} + +int ipc_close(ipch h, int fd) { + int i,r; + r=i=is_ipc_connection(h,fd); + if(i>=0) { + r=close(h->connection[i]); + h->connection[i]=-1; + h->connections--; + } + return r; +} + +int ipc_accept(ipch h) { + int i; + for(i=0;iconnection[i]==-1) break; + } + if(i==MAX_CONNECTIONS) return -2; + h->connection[i]=accept(h->listen_fd,NULL,NULL); + h->brdfilter[i]=0; + if(h->connection[i]>=0) h->connections++; + return h->connection[i]; +} + +int ipc_connect(char *name) { + int s,r; + struct sockaddr_un *socknaam; + socknaam=malloc(sizeof(*socknaam)); + if(!socknaam) return -2; + s=socket(PF_UNIX,SOCK_STREAM,0); + if(s<0) { + return s; + } + socknaam->sun_family=AF_UNIX; + strncpy(socknaam->sun_path,name,UNIX_PATH_MAX); + r=connect(s,(struct sockaddr *)socknaam,sizeof(*socknaam)); + if(r) { + close(s); + return -2; + } + return s; +} + +int ipc_sndmsg(int socket,void *msg, int msg_size) { + int *b,l; + if(msg_size<0) msg_size=strlen(msg)+1; + l=msg_size+2*sizeof(int); + b=alloca(l); + if(!b) return -1; + b[0]=IPCMAGIC; + b[1]=msg_size; + memcpy(&b[2],msg,msg_size); + /* atomic write */ + //ipc_check_closed(socket); + return write(socket,b,l); +} + +int ipc_set_filter(ipch h, int fd, int filter) { + int i,r; + r=i=is_ipc_connection(h,fd); + if(i>=0) { + r=h->brdfilter[i]; + h->brdfilter[i]=filter; + } + return r; +} +int ipc_brdmsg(ipch h,void *msg, int msg_size,int brdflg) { + int i,n; + for(n=i=0;iconnection[i]>=0 && (h->brdfilter[i]&brdflg)) { + ipc_sndmsg(h->connection[i],msg, msg_size); + n++; + } + } + return n; +} + +int ipc_rcvmsg(int socket, void *buf, int buf_size) { + int n,header[2]; + n=read(socket,header,sizeof(header)); + if(n!=sizeof(header)) return -2; + if(header[0]!=IPCMAGIC) return -3; + if(header[1]>buf_size) return -4; + n=read(socket,buf,header[1]); + if(n!=header[1]) return -5; + return header[1]; +} + +int ipc_rcvmsga(int socket, void **buf, int *buf_size) { + int n,header[2]; + void *buffer; + if(!buf) return -1; + n=read(socket,header,sizeof(header)); + if(n!=sizeof(header)) return -2; + if(header[0]!=IPCMAGIC) return -3; + /* Need to check a maximum amount */ + //if(header[1]>buf_size) return -4; + buffer=malloc(header[1]); + *buf=buffer; + /* Bad practice to not test the buf_size with protocol spec */ + if(buf_size) *buf_size=header[1]; + if(!buffer) return -4; + n=read(socket,buffer,header[1]); + if(n!=header[1]) return -5; + return header[1]; +} + diff -ruNp -X excludefiles trunk.409/ipc.h localwork/ipc.h --- trunk.409/ipc.h 1970-01-01 01:00:00.000000000 +0100 +++ localwork/ipc.h 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,25 @@ +#ifndef _IPC_H_ +#define _IPC_H_ +#define MAX_CONNECTIONS 16 +typedef struct { + int listen_fd; + int connections; + int connection[MAX_CONNECTIONS]; + int brdfilter[MAX_CONNECTIONS]; +} *ipch; +#define ipc_accept_fd(handle) (handle->listen_fd) +extern int ipc_set_fds(ipch ipc, fd_set *fds,int autoaccept); +extern int ipc_set_filter(ipch h, int fd, int filter); +extern int is_ipc_in_fds(ipch ipc, fd_set *fds,int autoaccept); +extern int ipc_accept(ipch h); +extern int is_ipc_connection(ipch h, int fd); +extern int ipc_sndmsg(int socket,void *msg, int msg_size); +extern int ipc_brdmsg(ipch h,void *msg, int msg_size,int brdflg); +extern int ipc_rcvmsg(int socket, void *buf, int buf_size); +extern int ipc_connect(char *name); +extern int ipc_close(ipch h,int fd); +extern ipch ipc_setup_listen(char *name); +extern int ipc_destroy(ipch h); +/* receive msg, alloc buffer on reception */ +extern int ipc_rcvmsga(int socket, void **buf, int *buf_size); +#endif diff -ruNp -X excludefiles trunk.409/ipc-listen.c localwork/ipc-listen.c --- trunk.409/ipc-listen.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/ipc-listen.c 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,58 @@ +#include +#include +/* bug in sys/un.h */ +#define UNIX_PATH_MAX 108 +#include +#include +#include +#include +#include +#include +#include "ipc.h" + +int quit=0; +#define CMP(x,y) (!strncmp(x,y,sizeof(y)-1)) +void handle_connection(ipch hserver, int c) { + int n; + char buffer[256]; + char buffer2[256]; + if((n=ipc_rcvmsg(c,buffer,sizeof(buffer)))<0) { + ipc_close(hserver,c); + return; + } + printf("Handling \"%s\" from %d\n",buffer,c); + if(CMP(buffer,"DOWN")) quit=1; + if(CMP(buffer,"EVENTS")) { + ipc_set_filter(hserver,c,1); + } + sprintf(buffer2,"OK: %s",buffer); + ipc_sndmsg(c,buffer2,-1); +} + +int main(int argc, char **argv) { + int n; + char buffer[256]; + ipch hserver; + fd_set rfds; + setvbuf(stdout,NULL,_IONBF,0); + FD_ZERO(&rfds); + signal(SIGPIPE,SIG_IGN); + hserver=ipc_setup_listen(argv[1]); + while(!quit) { + FD_SET(0,&rfds); + if(select(ipc_set_fds(hserver,&rfds,1)+1,&rfds,NULL,NULL,NULL)>0) { + int c; + if(FD_ISSET(0,&rfds)) { + FD_CLR(0,&rfds); + n=read(0,buffer,256); + buffer[n-1]=0; + printf("Broadcasting %s\n", buffer); + ipc_brdmsg(hserver,buffer,-1,1); + } + while((c=is_ipc_in_fds(hserver,&rfds,1))>=0) handle_connection(hserver,c); + } + } + ipc_destroy(hserver); +#define PUTS(x) write(0,x,strlen(x)); + return 0; +} diff -ruNp -X excludefiles trunk.409/ipc-send.c localwork/ipc-send.c --- trunk.409/ipc-send.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/ipc-send.c 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include +#include "ipc.h" + +int main(int argc, char **argv) { + int s,quit,r; + char buffer[256]; + fd_set rfds; + setvbuf(stdout,NULL,_IONBF,0); + s=ipc_connect(argv[1]); + FD_ZERO(&rfds); + for(quit=0;!quit;) { + FD_SET(s,&rfds); + FD_SET(0,&rfds); + bzero(buffer,256); + select(s+1,&rfds,NULL,NULL,NULL); + if(FD_ISSET(0,&rfds)) { + buffer[255]=0; + r=read(0,buffer,255); + if(r>0) { + buffer[r-1]=0; + ipc_sndmsg(s,buffer,-1); + } else { + quit=1; + } + } + if(FD_ISSET(s,&rfds)) { + buffer[255]=0; + r=ipc_rcvmsg(s,buffer,255); + if(r>=0) { + printf("GOT: %s\n",buffer); + } else { + quit=1; + } + } + } + close(s); + return 0; +} diff -ruNp -X excludefiles trunk.409/logger.c localwork/logger.c --- trunk.409/logger.c 1970-01-01 01:00:00.000000000 +0100 +++ localwork/logger.c 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,23 @@ +#include "csync2.h" +#include +#include + +int cslogopen(char *mode) { + openlog(mode,LOG_PID,LOG_DAEMON); + return 0; +} + +void cslog(int severity, const char *format) { + int prio; + va_list ap; + va_start(ap,format); + switch(severity) { + case CS_DEBUG: prio=LOG_DEBUG; break; + case CS_INFO: prio=LOG_INFO; break; + case CS_WARN: prio=LOG_WARN; break; + case CS_ERROR: prio=LOG_ERROR; break; + } + vsyslog(severity,format,ap); + va_end(ap); + return; +} diff -ruNp -X excludefiles trunk.409/Makefile.am localwork/Makefile.am --- trunk.409/Makefile.am 2009-11-24 18:04:53.000000000 +0100 +++ localwork/Makefile.am 2009-11-24 18:12:09.000000000 +0100 @@ -23,7 +23,8 @@ man_MANS = csync2.1 csync2_SOURCES = action.c cfgfile_parser.y cfgfile_scanner.l check.c \ checktxt.c csync2.c daemon.c db.c error.c getrealfn.c \ - groups.c rsync.c update.c urlencode.c conn.c prefixsubst.c + groups.c rsync.c update.c urlencode.c conn.c prefixsubst.c \ + csdb.c csdbd.c ipc.c AM_YFLAGS = -d BUILT_SOURCES = cfgfile_parser.h diff -ruNp -X excludefiles trunk.409/newdb.txt localwork/newdb.txt --- trunk.409/newdb.txt 1970-01-01 01:00:00.000000000 +0100 +++ localwork/newdb.txt 2009-11-24 18:12:09.000000000 +0100 @@ -0,0 +1,151 @@ +#################### +update.c +-------------------- + + SQL("Remove dirty-file entry.", + "DELETE FROM dirty WHERE filename = '%s' " + "AND peername = '%s'", url_encode(filename), + url_encode(peername)); + +csdbDeleteDirtyFilename_Peername_(const char *, const char *) + + +-------------------- + SQL_BEGIN("Get files for host from dirty table", + "SELECT filename, myname, force FROM dirty WHERE peername = '%s' " + "ORDER by filename ASC%s", url_encode(peername),limit_work) + { + const char *filename = url_decode(SQL_V(0)); + int i, use_this = patnum == 0; + for (i=0; i list +select random from list. +-------------------- +csync_insynctest + SQL_BEGIN("DB Dump - File", + "SELECT checktxt, filename FROM file %s%s%s ORDER BY filename", + filename ? "WHERE filename = '" : "", + filename ? url_encode(filename) : "", + filename ? "'" : "") + { + < Do a lot of stuff > + } SQL_END; + +-------------------- +csync_remove_old + SQL_BEGIN("Query dirty DB", + "SELECT filename, myname, peername FROM dirty") + while ((g=csync_find_next(g, filename)) != 0) { + + } SQL_END; + + SQL("Remove old file from dirty db", + "DELETE FROM dirty WHERE filename = '%s' AND peername = '%s'", t->value, t->value2); + +-------------------- + SQL_BEGIN("Query file DB", + "SELECT filename FROM file") + { + if (!csync_find_next(0, url_decode(SQL_V(0)))) + textlist_add(&tl, SQL_V(0), 0); + } SQL_END; +-------------------- + SQL("Remove old file from file db", + "DELETE FROM file WHERE filename = '%s'", t->value); +csdbFileDeleteFilename_(const char *) +-------------------- +#################### +check.c +-------------------- + SQL("Adding Hint", + "INSERT INTO hint (filename, recursive) " + "VALUES ('%s', %d)", url_encode(file), recursive); +csdbHintAddFilename_Recursive_() +-------------------- + SQL("Marking File Dirty", + "%s INTO dirty (filename, force, myname, peername) " + "VALUES ('%s', %s, '%s', '%s')", + csync_new_force ? "REPLACE" : "INSERT", + url_encode(file), + csync_new_force ? "1" : "0", + url_encode(pl[pl_idx].myname), + url_encode(pl[pl_idx].peername)); +csdbDirtyAddFilename_Myname_Peername_Force_() +-------------------- +csync_check_del + if ( !strcmp(file, "/") ) + asprintf(&where_rec, "or 1"); + else + asprintf(&where_rec, "UNION ALL SELECT filename from file where filename > '%s/' " + "and filename < '%s0'", + url_encode(file), url_encode(file)); + + SQL_BEGIN("Checking for removed files", + "SELECT filename from file where " + "filename = '%s' %s ORDER BY filename", url_encode(file), where_rec) + { + const char *filename = url_decode(SQL_V(0)); + if ( lstat_strict(prefixsubst(filename), &st) != 0 || csync_check_pure(filename) ) + textlist_add(&tl, filename, 0); + } SQL_END; +-------------------- +csync_check_mod + SQL_BEGIN("Checking File", + "SELECT checktxt FROM file WHERE " + "filename = '%s'", url_encode(file)) + { + if ( !csync_cmpchecktxt(checktxt, + url_decode(SQL_V(0))) ) { + csync_debug(2, "File has changed: %s\n", file); + this_is_dirty = 1; + } + } SQL_FIN { + if ( SQL_COUNT == 0 ) { + csync_debug(2, "New file: %s\n", file); + this_is_dirty = 1; + } + } SQL_END; + SQL("Adding or updating file entry", + "INSERT INTO file (filename, checktxt) " + "VALUES ('%s', '%s')", + url_encode(file), url_encode(checktxt)); + if (!init_run) csync_mark(file, 0, 0); + + + +================================================================================ +Daemon mode db server +-fork +-open databases +-accept/work loop + - accept->accept new fd + - wait on fd + - if fd readable: read instruction + +proto: + +SQL_BEGIN +SQL_NEXT +SQL_END +SQL_PLAIN +RSQL_ROW ... +RSQL_STATUS diff -ruNp -X excludefiles trunk.409/rsync.c localwork/rsync.c --- trunk.409/rsync.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/rsync.c 2009-11-24 18:12:09.000000000 +0100 @@ -25,10 +25,39 @@ #include #include +/* for tmpfile replacement: */ +#include +#include +#include + + #ifdef __CYGWIN__ #include #endif +#ifdef _SVID_SOURCE +static FILE *paranoid_tmpfile() +{ + char *name; + FILE *f; + int fd; + name=tempnam(csync_tempdir,"csync2"); + if(!name) { + csync_fatal("ERROR: tempnam() didn't return a valid filename!\n"); + } + f=NULL; + fd=open(name,O_CREAT|O_EXCL|O_RDWR,S_IWUSR|S_IRUSR); + if(fd>=0) { + f=fdopen(fd,"wb+"); + unlink(name); + } + if ( fd<0 || !f ) { + csync_fatal("ERROR: Could not open result from tempnam(%s)!\n",name); + } + free(name); + return f; +} +#else static FILE *paranoid_tmpfile() { FILE *f; @@ -41,6 +70,7 @@ static FILE *paranoid_tmpfile() return f; } +#endif void csync_send_file(FILE *in) { diff -ruNp -X excludefiles trunk.409/update.c localwork/update.c --- trunk.409/update.c 2009-11-24 18:04:53.000000000 +0100 +++ localwork/update.c 2009-11-24 18:12:09.000000000 +0100 @@ -212,10 +212,7 @@ auto_resolve_entry_point: goto maybe_auto_resolve; skip_action: - SQL("Remove dirty-file entry.", - "DELETE FROM dirty WHERE filename = '%s' " - "AND peername = '%s'", url_encode(filename), - url_encode(peername)); + csdbDeleteDirtyFilename_Peername_(url_encode(filename),url_encode(peername)); if (auto_resolve_run) csync_error_count--; @@ -426,10 +423,7 @@ skip_action: goto got_error; } - SQL("Remove dirty-file entry.", - "DELETE FROM dirty WHERE filename = '%s' " - "AND peername = '%s'", url_encode(filename), - url_encode(peername)); + csdbDeleteDirtyFilename_Peername_(url_encode(filename),url_encode(peername)); if (auto_resolve_run) csync_error_count--; @@ -536,22 +530,22 @@ int compare_files(const char *filename, void csync_update_host(const char *peername, const char **patlist, int patnum, int recursive, int dry_run) { + struct textlist *dirtylist; struct textlist *tl = 0, *t, *next_t; struct textlist *tl_mod = 0, **last_tn=&tl; char *current_name = 0; struct stat st; - SQL_BEGIN("Get files for host from dirty table", - "SELECT filename, myname, force FROM dirty WHERE peername = '%s' " - "ORDER by filename ASC", url_encode(peername)) - { - const char *filename = url_decode(SQL_V(0)); + dirtylist=csdbGetDirtyFilesPeername_Maxamount_(url_encode(peername),csync_limit_work); + for(t=dirtylist;t;t=t->next) { int i, use_this = patnum == 0; + const char *filename = url_decode(t->value); for (i=0; ivalue2), t->intvalue); + } + textlist_free(dirtylist); /* just return if there are no files to update */ if ( !tl ) return; @@ -623,12 +617,7 @@ void csync_update(const char ** patlist, { struct textlist *tl = 0, *t; - SQL_BEGIN("Get hosts from dirty table", - "SELECT peername FROM dirty GROUP BY peername ORDER BY random()") - { - textlist_add(&tl, url_decode(SQL_V(0)), 0); - } SQL_END; - + tl=csdbGetPeers(); for (t = tl; t != 0; t = t->next) { if (active_peerlist) { int i=0, pnamelen = strlen(t->value); @@ -755,7 +744,7 @@ int csync_insynctest_readline(char **fil int csync_insynctest(const char *myname, const char *peername, int init_run, int auto_diff, const char *filename) { - struct textlist *diff_list = 0, *diff_ent; + struct textlist *cf,*filelist=0,*diff_list = 0, *diff_ent; const struct csync_group *g; const struct csync_group_host *h; char *r_file=0, *r_checktxt=0; @@ -792,13 +781,10 @@ found_host: } conn_printf("\n"); - SQL_BEGIN("DB Dump - File", - "SELECT checktxt, filename FROM file %s%s%s ORDER BY filename", - filename ? "WHERE filename = '" : "", - filename ? url_encode(filename) : "", - filename ? "'" : "") - { - char *l_file = strdup(url_decode(SQL_V(1))), *l_checktxt = strdup(url_decode(SQL_V(0))); + + filelist=csdbGetFilesFilename_(filename?url_encode(filename):filename); + for(cf=filelist;cf;cf=cf->next) { + char *l_file = strdup(url_decode(cf->value2)), *l_checktxt = strdup(url_decode(cf->value)); if ( csync_match_file_host(l_file, myname, peername, 0) ) { if ( remote_eof ) { got_remote_eof: @@ -847,7 +833,8 @@ got_remote_eof: } free(l_checktxt); free(l_file); - } SQL_END; + } + textlist_free(filelist); if ( !remote_eof ) while ( !csync_insynctest_readline(&r_file, &r_checktxt) ) { @@ -929,46 +916,45 @@ skip_this_peername: ; void csync_remove_old() { struct textlist *tl = 0, *t; + struct textlist *dirtylist, *filelist; + dirtylist=csdbGetDirty(); - SQL_BEGIN("Query dirty DB", - "SELECT filename, myname, peername FROM dirty") - { + for(t=dirtylist;t;t=t->next) { const struct csync_group *g = 0; const struct csync_group_host *h; - const char *filename = url_decode(SQL_V(0)); + const char *filename = url_decode(t->value); while ((g=csync_find_next(g, filename)) != 0) { - if (!strcmp(g->myname, SQL_V(1))) + if (!strcmp(g->myname, t->value2)) for (h = g->host; h; h = h->next) { - if (!strcmp(h->hostname, SQL_V(2))) + if (!strcmp(h->hostname, t->value3)) goto this_dirty_record_is_ok; } } - textlist_add2(&tl, SQL_V(0), SQL_V(2), 0); + textlist_add2(&tl, t->value, t->value3, 0); this_dirty_record_is_ok: ; - } SQL_END; + } + textlist_free(dirtylist); for (t = tl; t != 0; t = t->next) { csync_debug(1, "Removing %s (%s) from dirty db.\n", t->value, t->value2); - SQL("Remove old file from dirty db", - "DELETE FROM dirty WHERE filename = '%s' AND peername = '%s'", t->value, t->value2); + csdbDeleteDirtyFilename_Peername_(t->value, t->value2); } textlist_free(tl); tl = 0; - SQL_BEGIN("Query file DB", - "SELECT filename FROM file") - { - if (!csync_find_next(0, url_decode(SQL_V(0)))) - textlist_add(&tl, SQL_V(0), 0); - } SQL_END; + filelist=csdbGetFilesNamesonly(); + for(t=filelist;t;t=t->next) { + if (!csync_find_next(0, url_decode(t->value))) + textlist_add(&tl, t->value, 0); + } + textlist_free(filelist); for (t = tl; t != 0; t = t->next) { csync_debug(1, "Removing %s from file db.\n", t->value); - SQL("Remove old file from file db", - "DELETE FROM file WHERE filename = '%s'", t->value); + csdbDeleteFileFilename_(t->value); } textlist_free(tl); }