NetRender:

- multires cache files and image .tex cache support in dependency list
- Compare md5 of files before using a local copy (not one transfered by netrender). Could be changed to a simpler CRC if speed is an issue. The goal is not to have a strong crypto signature but just to detect outdated local files.
- Reduce slave timeout to 5 minutes (down from 30). Slaves should report at most every 30s, there's no reason for a value to be that high.
- Reorder the presentation tables on the main web page (job list is more important)
- Collapse dependency list by default on job page (only show main file and headers for other files, point cache and fluid cache)
- Slave option (default: True) to also output render log to the console (as well as the usual copy to the master)

release/scripts/io/netrender/client.py

@@ -125,6 +125,10 @@ def clientSendJob(conn, scene, anim = False):
             file_path = bpy.utils.expandpath(image.filename)
             if os.path.exists(file_path):
                 job.addFile(file_path)
+                
+                tex_path = os.path.splitext(file_path)[0] + ".tex"
+                if os.path.exists(tex_path):
+                    job.addFile(tex_path)
 
     ###########################
     # FLUID + POINT CACHE
@@ -144,6 +148,9 @@ def clientSendJob(conn, scene, anim = False):
                 addPointCache(job, object, modifier.domain_settings.point_cache_low, default_path)
                 if modifier.domain_settings.highres:
                     addPointCache(job, object, modifier.domain_settings.point_cache_high, default_path)
+            elif modifier.type == "MULTIRES" and modifier.external:
+                file_path = bpy.utils.expandpath(modifier.filename)
+                job.addFile(file_path)
 
         # particles modifier are stupid and don't contain data
         # we have to go through the object property

release/scripts/io/netrender/master.py

@@ -27,12 +27,16 @@ import netrender.balancing
 import netrender.master_html
 
 class MRenderFile(netrender.model.RenderFile):
-    def __init__(self, filepath, index, start, end):
-        super().__init__(filepath, index, start, end)
+    def __init__(self, filepath, index, start, end, signature):
+        super().__init__(filepath, index, start, end, signature)
         self.found = False
 
     def test(self):
         self.found = os.path.exists(self.filepath)
+        if self.found:
+            found_signature = hashFile(self.filepath)
+            self.found = self.signature == found_signature
+            
         return self.found
 
 
@@ -74,7 +78,7 @@ class MRenderJob(netrender.model.RenderJob):
         # special server properties
         self.last_update = 0
         self.save_path = ""
-        self.files = [MRenderFile(rfile.filepath, rfile.index, rfile.start, rfile.end) for rfile in job_info.files]
+        self.files = [MRenderFile(rfile.filepath, rfile.index, rfile.start, rfile.end, rfile.signature) for rfile in job_info.files]
 
         self.resolution = None
 
@@ -716,7 +720,7 @@ class RenderHandler(http.server.BaseHTTPRequestHandler):
                         buf = self.rfile.read(length)
 
                         # add same temp file + renames as slave
-
+                        
                         f = open(file_path, "wb")
                         f.write(buf)
                         f.close()
@@ -875,7 +879,7 @@ class RenderMasterServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
         self.job_id = 0
         self.path = path + "master_" + str(os.getpid()) + os.sep
 
-        self.slave_timeout = 30 # 30 mins: need a parameter for that
+        self.slave_timeout = 5 # 5 mins: need a parameter for that
 
         self.balancer = netrender.balancing.Balancer()
         self.balancer.addRule(netrender.balancing.RatingUsageByCategory(self.getJobs))

release/scripts/io/netrender/master_html.py

@@ -106,53 +106,6 @@ def get(handler):
         handler.send_head(content = "text/html")
         head("NetRender")
 
-        output("<h2>Master</h2>")
-
-        output("""<button title="remove all jobs" onclick="clear_jobs();">CLEAR JOB LIST</button>""")
-
-        startTable(caption = "Rules", class_style = "rules")
-
-        headerTable("type", "enabled", "description", "limit")
-
-        for rule in handler.server.balancer.rules:
-            rowTable(
-                        "rating",
-                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
-                        rule,
-                        rule.str_limit() +
-                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
-                    )
-
-        for rule in handler.server.balancer.priorities:
-            rowTable(
-                        "priority",
-                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
-                        rule,
-                        rule.str_limit() +
-                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
-                    )
-
-        for rule in handler.server.balancer.exceptions:
-            rowTable(
-                        "exception",
-                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
-                        rule,
-                        rule.str_limit() +
-                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
-                    )
-
-        endTable()
-
-        output("<h2>Slaves</h2>")
-
-        startTable()
-        headerTable("name", "address", "last seen", "stats", "job")
-
-        for slave in handler.server.slaves:
-            rowTable(slave.name, slave.address[0], time.ctime(slave.last_seen), slave.stats, link(slave.job.name, "/html/job" + slave.job.id) if slave.job else "None")
-
-        endTable()
-
         output("<h2>Jobs</h2>")
 
         startTable()
@@ -204,6 +157,53 @@ def get(handler):
                     )
 
         endTable()
+        
+        output("<h2>Slaves</h2>")
+
+        startTable()
+        headerTable("name", "address", "last seen", "stats", "job")
+
+        for slave in handler.server.slaves:
+            rowTable(slave.name, slave.address[0], time.ctime(slave.last_seen), slave.stats, link(slave.job.name, "/html/job" + slave.job.id) if slave.job else "None")
+
+        endTable()
+
+        output("<h2>Configuration</h2>")
+
+        output("""<button title="remove all jobs" onclick="clear_jobs();">CLEAR JOB LIST</button>""")
+
+        startTable(caption = "Rules", class_style = "rules")
+
+        headerTable("type", "enabled", "description", "limit")
+
+        for rule in handler.server.balancer.rules:
+            rowTable(
+                        "rating",
+                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
+                        rule,
+                        rule.str_limit() +
+                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
+                    )
+
+        for rule in handler.server.balancer.priorities:
+            rowTable(
+                        "priority",
+                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
+                        rule,
+                        rule.str_limit() +
+                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
+                    )
+
+        for rule in handler.server.balancer.exceptions:
+            rowTable(
+                        "exception",
+                        checkbox("", rule.enabled, "balance_enable('%i', '%s')" % (id(rule), str(not rule.enabled))),
+                        rule,
+                        rule.str_limit() +
+                        """<button title="edit limit" onclick="balance_edit('%i', '%s');">edit</button>""" % (id(rule), str(rule.limit)) if hasattr(rule, "limit") else "&nbsp;"
+                    )
+
+        endTable()
 
         output("</body></html>")
 
@@ -235,13 +235,17 @@ def get(handler):
             tot_cache = 0
             tot_fluid = 0
 
+            rowTable(job.files[0].filepath)
+            rowTable("Other Files", class_style = "toggle", extra = "onclick='toggleDisplay(&quot;.other&quot;, &quot;none&quot;, &quot;table-row&quot;)'")
+
             for file in job.files:
                 if file.filepath.endswith(".bphys"):
                     tot_cache += 1
                 elif file.filepath.endswith(".bobj.gz") or file.filepath.endswith(".bvel.gz"):
                     tot_fluid += 1
                 else:
-                    rowTable(file.filepath)
+                    if file != job.files[0]:
+                        rowTable(file.filepath, class_style = "other")
 
             if tot_cache > 0:
                 rowTable("%i physic cache files" % tot_cache, class_style = "toggle", extra = "onclick='toggleDisplay(&quot;.cache&quot;, &quot;none&quot;, &quot;table-row&quot;)'")
@@ -257,9 +261,9 @@ def get(handler):
 
             endTable()
 
-            output("<h2>Blacklist</h2>")
-
             if job.blacklist:
+                output("<h2>Blacklist</h2>")
+
                 startTable()
                 headerTable("name", "address")
 
@@ -268,8 +272,6 @@ def get(handler):
                     rowTable(slave.name, slave.address[0])
 
                 endTable()
-            else:
-                output("<i>Empty</i>")
 
             output("<h2>Frames</h2>")
 

release/scripts/io/netrender/model.py

@@ -103,8 +103,9 @@ JOB_TYPES = {
                         }
 
 class RenderFile:
-    def __init__(self, filepath = "", index = 0, start = -1, end = -1):
+    def __init__(self, filepath = "", index = 0, start = -1, end = -1, signature=0):
         self.filepath = filepath
+        self.signature = signature
         self.index = index
         self.start = start
         self.end = end
@@ -114,7 +115,8 @@ class RenderFile:
                     "filepath": self.filepath,
                     "index": self.index,
                     "start": self.start,
-                    "end": self.end
+                    "end": self.end,
+                    "signature": self.signature
                 }
 
     @staticmethod
@@ -122,7 +124,7 @@ class RenderFile:
         if not data:
             return None
 
-        rfile = RenderFile(data["filepath"], data["index"], data["start"], data["end"])
+        rfile = RenderFile(data["filepath"], data["index"], data["start"], data["end"], data["signature"])
 
         return rfile
 
@@ -153,7 +155,8 @@ class RenderJob:
             self.blacklist = job_info.blacklist
 
     def addFile(self, file_path, start=-1, end=-1):
-        self.files.append(RenderFile(file_path, len(self.files), start, end))
+        signature = hashFile(file_path)
+        self.files.append(RenderFile(file_path, len(self.files), start, end, signature))
 
     def addFrame(self, frame_number, command = ""):
         frame = RenderFrame(frame_number, command)

release/scripts/io/netrender/netrender.css

@@ -68,6 +68,10 @@ button {
 	display: none;
 }
 
+.other {
+	display: none;
+}
+
 .rules {
 	width: 60em;
 	text-align: left;

release/scripts/io/netrender/slave.py

@@ -64,12 +64,21 @@ def testCancel(conn, job_id, frame_number):
         else:
             return False
 
-def testFile(conn, job_id, slave_id, file_index, JOB_PREFIX, file_path, main_path = None):
-    job_full_path = prefixPath(JOB_PREFIX, file_path, main_path)
+def testFile(conn, job_id, slave_id, rfile, JOB_PREFIX, main_path = None):
+    job_full_path = prefixPath(JOB_PREFIX, rfile.filepath, main_path)
+    
+    found = os.path.exists(job_full_path)
+    
+    if found:
+        found_signature = hashFile(job_full_path)
+        found = found_signature == rfile.signature
+        
+        if not found:
+            print("Found file %s at %s but signature mismatch!" % (rfile.filepath, job_full_path))
 
-    if not os.path.exists(job_full_path):
+    if not found:
         temp_path = JOB_PREFIX + "slave.temp.blend"
-        conn.request("GET", fileURL(job_id, file_index), headers={"slave-id":slave_id})
+        conn.request("GET", fileURL(job_id, rfile.index), headers={"slave-id":slave_id})
         response = conn.getresponse()
 
         if response.status != http.client.OK:
@@ -126,14 +135,14 @@ def render_slave(engine, netsettings, threads):
                     job_path = job.files[0].filepath # path of main file
                     main_path, main_file = os.path.split(job_path)
 
-                    job_full_path = testFile(conn, job.id, slave_id, 0, JOB_PREFIX, job_path)
+                    job_full_path = testFile(conn, job.id, slave_id, job.files[0], JOB_PREFIX)
                     print("Fullpath", job_full_path)
                     print("File:", main_file, "and %i other files" % (len(job.files) - 1,))
                     engine.update_stats("", "Render File "+ main_file+ " for job "+ job.id)
 
                     for rfile in job.files[1:]:
                         print("\t", rfile.filepath)
-                        testFile(conn, job.id, slave_id, rfile.index, JOB_PREFIX, rfile.filepath, main_path)
+                        testFile(conn, job.id, slave_id, rfile, JOB_PREFIX, main_path)
 
                 # announce log to master
                 logfile = netrender.model.LogFile(job.id, slave_id, [frame.number for frame in job.frames])
@@ -178,6 +187,10 @@ def render_slave(engine, netsettings, threads):
                             # (only need to update on one frame, they are linked
                             conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers)
                             response = conn.getresponse()
+                            
+                            # Also output on console
+                            if netsettings.slave_thumb:
+                                print(str(stdout, encoding='utf8'), end="")
 
                             stdout = bytes()
 
@@ -194,6 +207,17 @@ def render_slave(engine, netsettings, threads):
                         process.terminate()
                     continue # to next frame
 
+                # flush the rest of the logs
+                if stdout:
+                    # Also output on console
+                    if netsettings.slave_thumb:
+                        print(str(stdout, encoding='utf8'), end="")
+                    
+                    # (only need to update on one frame, they are linked
+                    conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers)
+                    if conn.getresponse().status == http.client.NO_CONTENT:
+                        continue
+
                 total_t = time.time() - start_t
 
                 avg_t = total_t / len(job.frames)
@@ -202,13 +226,6 @@ def render_slave(engine, netsettings, threads):
 
                 print("status", status)
 
-                # flush the rest of the logs
-                if stdout:
-                    # (only need to update on one frame, they are linked
-                    conn.request("PUT", logURL(job.id, first_frame), stdout, headers=headers)
-                    if conn.getresponse().status == http.client.NO_CONTENT:
-                        continue
-
                 headers = {"job-id":job.id, "slave-id":slave_id, "job-time":str(avg_t)}
 
 

release/scripts/io/netrender/ui.py

@@ -76,7 +76,7 @@ def verify_address(netsettings):
         else:
             netsettings.server_address = "[default]"
 
-class RenderButtonsPanel(bpy.types.Panel):
+class RenderButtonsPanel():
     bl_space_type = "PROPERTIES"
     bl_region_type = "WINDOW"
     bl_context = "render"
@@ -88,7 +88,7 @@ class RenderButtonsPanel(bpy.types.Panel):
 
 # Setting panel, use in the scene for now.
 @rnaType
-class RENDER_PT_network_settings(RenderButtonsPanel):
+class RENDER_PT_network_settings(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Network Settings"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -123,7 +123,7 @@ class RENDER_PT_network_settings(RenderButtonsPanel):
         layout.operator("render.netclientweb", icon='QUESTION')
 
 @rnaType
-class RENDER_PT_network_slave_settings(RenderButtonsPanel):
+class RENDER_PT_network_slave_settings(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Slave Settings"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -141,13 +141,14 @@ class RENDER_PT_network_slave_settings(RenderButtonsPanel):
 
         layout.prop(netsettings, "slave_clear")
         layout.prop(netsettings, "slave_thumb")
+        layout.prop(netsettings, "slave_outputlog")
         layout.label(text="Threads:")
         layout.prop(rd, "threads_mode", expand=True)
         sub = layout.column()
         sub.enabled = rd.threads_mode == 'FIXED'
         sub.prop(rd, "threads")
 @rnaType
-class RENDER_PT_network_master_settings(RenderButtonsPanel):
+class RENDER_PT_network_master_settings(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Master Settings"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -166,7 +167,7 @@ class RENDER_PT_network_master_settings(RenderButtonsPanel):
         layout.prop(netsettings, "master_clear")
 
 @rnaType
-class RENDER_PT_network_job(RenderButtonsPanel):
+class RENDER_PT_network_job(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Job Settings"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -206,7 +207,7 @@ class RENDER_PT_network_job(RenderButtonsPanel):
         row.prop(netsettings, "chunks")
 
 @rnaType
-class RENDER_PT_network_slaves(RenderButtonsPanel):
+class RENDER_PT_network_slaves(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Slaves Status"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -245,7 +246,7 @@ class RENDER_PT_network_slaves(RenderButtonsPanel):
             layout.label(text="Stats: " + slave.stats)
 
 @rnaType
-class RENDER_PT_network_slaves_blacklist(RenderButtonsPanel):
+class RENDER_PT_network_slaves_blacklist(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Slaves Blacklist"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -283,7 +284,7 @@ class RENDER_PT_network_slaves_blacklist(RenderButtonsPanel):
             layout.label(text="Stats: " + slave.stats)
 
 @rnaType
-class RENDER_PT_network_jobs(RenderButtonsPanel):
+class RENDER_PT_network_jobs(bpy.types.Panel, RenderButtonsPanel):
     bl_label = "Jobs"
     COMPAT_ENGINES = {'NET_RENDER'}
 
@@ -365,6 +366,11 @@ NetRenderSettings.BoolProperty( attr="slave_thumb",
                 description="Generate thumbnails on slaves instead of master",
                 default = False)
 
+NetRenderSettings.BoolProperty( attr="slave_outputlog",
+                name="Output render log on console",
+                description="Output render text log to console as well as sending it to the master",
+                default = True)
+
 NetRenderSettings.BoolProperty( attr="master_clear",
                 name="Clear on exit",
                 description="delete saved files on exit",

release/scripts/io/netrender/utils.py

@@ -19,7 +19,7 @@
 import sys, os
 import re
 import http, http.client, http.server, urllib, socket
-import subprocess, shutil, time, hashlib
+import subprocess, shutil, time, hashlib, zlib
 
 import netrender.model
 
@@ -154,6 +154,18 @@ def renderURL(job_id, frame_number):
 def cancelURL(job_id):
     return "/cancel_%s" % (job_id)
 
+def hashFile(path):
+    f = open(path, "rb")
+    value = hashData(f.read())
+    f.close()
+    return value
+    
+def hashData(data):
+    m = hashlib.md5()
+    m.update(data)
+    return m.hexdigest()
+    
+
 def prefixPath(prefix_directory, file_path, prefix_path):
     if os.path.isabs(file_path):
         # if an absolute path, make sure path exists, if it doesn't, use relative local path