Net/ftp is cpu bound on fast internet connections

ruby-talk
1

Hi,

I was trying to run multiple ftp.putbinaryfile jobs and noticed that if
I run more than say 2 threads on my PIII 933, My CPU usage hits 100% and
my IO stays about the same, even though I am nowhere close to my
theorectical limit. Here’s the original code.

1.upto(5) {|server_suffix|
puts “starting send to DL360#{server_suffix}”
threads[server_suffix] = Thread.new {
ftp[server_suffix].putbinaryfile ‘100recs.tar.gz’,
‘100recs.tar.gz’

     telnet[server_suffix].cmd 'tar -xzf 100recs.tar.gz'

}
}
threads[1…5].each {|thread| thread.join}

My connection should actually be able to handle about 20 of these
threads at a time. What I noticed was that of course ftp.rb’s routines
are block oriented and pass everything through yield to a block for
processing. I was wondering if it would be good for me or someone to
add non block-oriented functions that perhaps even memory map and or
buffer files where appropriate to save cpu time.

Can you try this patch?

Shugo

Index: lib/net/ftp.rb

···

===================================================================
RCS file: /src/ruby/lib/net/ftp.rb,v
retrieving revision 1.10
diff -u -r1.10 ftp.rb
— lib/net/ftp.rb 2002/06/11 04:20:11 1.10
+++ lib/net/ftp.rb 2002/06/11 08:35:51
@@ -261,26 +261,21 @@
@welcome = resp
end

  • def retrbinary(cmd, blocksize, rest_offset = nil, callback = Proc.new)
  • def retrbinary(cmd, blocksize, rest_offset = nil)
    synchronize do
    voidcmd(“TYPE I”)
    conn = transfercmd(cmd, rest_offset)
    loop do
    data = conn.read(blocksize)
    break if data == nil
  • callback.call(data)

  • yield(data)
    end
    conn.close
    voidresp
    end
    end
  • def retrlines(cmd, callback = nil)
  •  if block_given?
  • callback = Proc.new
  •  elsif not callback.is_a?(Proc)
  • callback = Proc.new {|line| print line, “\n”}
  •  end
  • def retrlines(cmd)
    synchronize do
    voidcmd(“TYPE A”)
    conn = transfercmd(cmd)
    @@ -292,18 +287,14 @@
    elsif line[-1] == ?\n
    line = line[0 … -2]
    end
  • callback.call(line)

  • yield(line)
    end
    conn.close
    voidresp
    end
    end
  • def storbinary(cmd, file, blocksize, rest_offset = nil, callback = nil)
  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)
  • def storbinary(cmd, file, blocksize, rest_offset = nil, &block)
    synchronize do
    voidcmd(“TYPE I”)
    conn = transfercmd(cmd, rest_offset)
    @@ -311,18 +302,14 @@
    buf = file.read(blocksize)
    break if buf == nil
    conn.write(buf)
  • callback.call(buf) if use_callback

  • yield(buf) if block
    end
    conn.close
    voidresp
    end
    end
  • def storlines(cmd, file, callback = nil)
  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)
  • def storlines(cmd, file, &block)
    synchronize do
    voidcmd(“TYPE A”)
    conn = transfercmd(cmd)
    @@ -333,7 +320,7 @@
    buf = buf.chomp + CRLF
    end
    conn.write(buf)
  • callback.call(buf) if use_callback

  • yield(buf) if block

    end
    conn.close
    voidresp
    @@ -341,11 +328,7 @@
    end

    def getbinaryfile(remotefile, localfile,

  •         blocksize = DEFAULT_BLOCKSIZE, callback = nil)

  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)

  •         blocksize = DEFAULT_BLOCKSIZE, &block)
 if @resume
    rest_offset = File.size?(localfile)
    f = open(localfile, “a”)
    @@ -357,24 +340,20 @@
    f.binmode
    retrbinary("RETR " + remotefile, blocksize, rest_offset) do |data|
    f.write(data)
  • callback.call(data) if use_callback

  • yield(data) if block
    end
    ensure
    f.close
    end
    end
  • def gettextfile(remotefile, localfile, callback = nil)
  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)
  • def gettextfile(remotefile, localfile, &block)
    f = open(localfile, “w”)
    begin
    retrlines("RETR " + remotefile) do |line|
    line = line + @return_code
    f.write(line)
  • callback.call(line) if use_callback

  • yield(line) if block

    end
    ensure
    f.close
    @@ -382,11 +361,7 @@
    end

    def putbinaryfile(localfile, remotefile,

  •         blocksize = DEFAULT_BLOCKSIZE, callback = nil)

  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)

  •         blocksize = DEFAULT_BLOCKSIZE, &block)
 if @resume
    rest_offset = size(remotefile)
    else
    @@ -395,24 +370,16 @@
    f = open(localfile)
    begin
    f.binmode
  • storbinary("STOR " + remotefile, f, blocksize, rest_offset) do |data|
  • callback.call(data) if use_callback
  • end
  • storbinary("STOR " + remotefile, f, blocksize, rest_offset, &block)
    ensure
    f.close
    end
    end
  • def puttextfile(localfile, remotefile, callback = nil)
  •  if block_given?
  • callback = Proc.new
  •  end

  •  use_callback = callback.is_a?(Proc)
  • def puttextfile(localfile, remotefile, &block)
    f = open(localfile)
    begin
  • storlines("STOR " + remotefile, f) do |line|
  • callback.call(line) if use_callback
  • end
  • storlines("STOR " + remotefile, f, &block)
    ensure
    f.close
    end