Re: [nfsv4] [PATCH 2/3] Morph LAYOUTRETURN errors into LAYOUTERROR

[Benny Halevy <[e-mail]>]

On 05/15/2014 08:56 PM, Thomas Haynes wrote:
> From: Tom Haynes <[e-mail]>
>
> Signed-off-by: Tom Haynes <[e-mail]>
> ---
>  Makefile                          |   7 +-
>  dotx.d/Makefile                   |   6 +-
>  dotx.d/fore_prog.x                |   6 --
>  dotx.d/layout_errors.x            |  14 ---
>  dotx.d/layouterror_args.x         |   8 ++
>  dotx.d/layouterror_res.x          |   5 ++
>  dotx.d/spit_types.sh              |  15 ----
>  dotx.d/type_device_error.x        |   7 ++
>  nfsv42_middle_errors.xml          |  31 +++++++
>  nfsv42_middle_new_errors.xml      |  31 +++++++
>  nfsv42_middle_op_layouterror.xml  | 147 ++++++++++++++++++++++++++++++
>  nfsv42_middle_op_layoutreturn.xml | 185 --------------------------------------
>  nfsv42_middle_op_layoutstats.xml  |   2 +-
>  nfsv42_middle_op_mandlist.xml     |   3 +-
>  14 files changed, 240 insertions(+), 227 deletions(-)
>  delete mode 100644 dotx.d/layout_errors.x
>  create mode 100644 dotx.d/layouterror_args.x
>  create mode 100644 dotx.d/layouterror_res.x
>  create mode 100644 dotx.d/type_device_error.x
>  create mode 100644 nfsv42_middle_op_layouterror.xml
>  delete mode 100644 nfsv42_middle_op_layoutreturn.xml
>
> diff --git a/Makefile b/Makefile
> index 3a81c8f..669d0dd 100755
> --- a/Makefile
> +++ b/Makefile
> @@ -125,7 +125,6 @@ SPITGEN =	dotx.d/type_nfstime4.x \
>  		dotx.d/type_ssv_seal_plain_tkn4.x \
>  		dotx.d/type_ssv_seal_cipher_tkn4.x \
>  		dotx.d/type_layoutreturn4.x \
> -		dotx.d/type_layoutreturn_errs.x \
>  		dotx.d/type_client_owner4.x \
>  		dotx.d/type_server_owner4.x \
>  		dotx.d/type_device_addr4.x \
> @@ -203,7 +202,6 @@ SPITGENXML =	autogen/type_nfstime4.xml \
>  		autogen/type_ssv_seal_plain_tkn4.xml \
>  		autogen/type_ssv_seal_cipher_tkn4.xml \
>  		autogen/type_layoutreturn4.xml \
> -		autogen/type_layoutreturn_errs.xml \
>  		autogen/type_client_owner4.xml \
>  		autogen/type_server_owner4.xml \
>  		autogen/type_device_addr4.xml \
> @@ -351,6 +349,9 @@ AUTOGEN =	\
>  		autogen/io_advise_res.xml \
>  		autogen/layoutcommit_args.xml \
>  		autogen/layoutcommit_res.xml \
> +		autogen/type_device_error.xml \
> +		autogen/layouterror_args.xml \
> +		autogen/layouterror_res.xml \
>  		autogen/layoutget_args.xml \
>  		autogen/layoutget_res.xml \
>  		autogen/layoutreturn_args.xml \
> @@ -450,7 +451,7 @@ IDXMLSRC_BASE = \
>  	${DOC_PREFIX}_middle_op_exchange_id.xml \
>  	${DOC_PREFIX}_middle_op_deallocate.xml \
>  	${DOC_PREFIX}_middle_op_io_advise.xml \
> -	${DOC_PREFIX}_middle_op_layoutreturn.xml \
> +	${DOC_PREFIX}_middle_op_layouterror.xml \
>  	${DOC_PREFIX}_middle_op_layoutstats.xml \
>  	${DOC_PREFIX}_middle_op_offload_cancel.xml \
>  	${DOC_PREFIX}_middle_op_offload_status.xml \
> diff --git a/dotx.d/Makefile b/dotx.d/Makefile
> index 555b9ee..d2e81c3 100755
> --- a/dotx.d/Makefile
> +++ b/dotx.d/Makefile
> @@ -123,6 +123,9 @@ FORE_OPS_AS_IS2 = \
>  		type_io_advise.x \
>  		io_advise_args.x \
>  		io_advise_res.x \
> +		type_device_error.x \
> +		layouterror_args.x \
> +		layouterror_res.x \
>  		type_io_info.x \
>  		layoutstats_args.x \
>  		layoutstats_res.x \
> @@ -196,7 +199,6 @@ SPITGEN =	basic_types.x \
>  		type_ssv_seal_plain_tkn4.x \
>  		type_ssv_seal_cipher_tkn4.x \
>  		type_layoutreturn4.x \
> -		type_layoutreturn_errs.x \
>  		type_client_owner4.x \
>  		type_server_owner4.x \
>  		type_device_addr4.x \
> @@ -255,7 +257,7 @@ AUTOGEN =	hand_edit_warning.x \
>  
>  all:	nfsv42.x filelist
>  
> -nfsv42.x: $(AUTOGEN) hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x layout_errors.x fore_prog.x \
> +nfsv42.x: $(AUTOGEN) hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x fore_prog.x \
>  		back_ops.x back_prog.x
>  	m4 hand_edit_warning.x head.x fs_locations_info.x file_layout.x fore_ops.x fore_prog.x back_ops.x back_prog.x | expand | tr -d "\r" > nfsv42.tmp
>  	mv nfsv42.tmp nfsv42.x
> diff --git a/dotx.d/fore_prog.x b/dotx.d/fore_prog.x
> index 5eb336e..d0b92c6 100644
> --- a/dotx.d/fore_prog.x
> +++ b/dotx.d/fore_prog.x
> @@ -10,12 +10,6 @@ include(type_COMPOUND4args.x)
>  include(type_COMPOUND4res.x)
>  
>  /*
> - * Layout return errors, which might
> - * include the nfs_opnum4.
> - */
> -include(layout_errors.x)
> -
> -/*
>   * Remote file service routines
>   */
>  program NFS4_PROGRAM {
> diff --git a/dotx.d/layout_errors.x b/dotx.d/layout_errors.x
> deleted file mode 100644
> index 2ed5df8..0000000
> --- a/dotx.d/layout_errors.x
> +++ /dev/null
> @@ -1,14 +0,0 @@
> -
> -%/*
> -% * Encoded in the lou_body field of data type layoutupdate4:
> -% *	 Nothing. lou_body is a zero length array of bytes.
> -% */
> -%
> -
> -%/*
> -% * Encoded in the lrf_body field of
> -% * data type layoutreturn_file4:
> -% */
> -include(type_layoutreturn_errs.x)
> -%
> -
> diff --git a/dotx.d/layouterror_args.x b/dotx.d/layouterror_args.x
> new file mode 100644
> index 0000000..8d1fcaf
> --- /dev/null
> +++ b/dotx.d/layouterror_args.x
> @@ -0,0 +1,8 @@
> +
> +struct LAYOUTERROR4args {
> +        /* CURRENT_FH: file */
> +        offset4			lea_offset;
> +        length4			lea_length;
> +        stateid4		lea_stateid;
> +        device_error4		lea_errors;
> +};
> diff --git a/dotx.d/layouterror_res.x b/dotx.d/layouterror_res.x
> new file mode 100644
> index 0000000..921348a
> --- /dev/null
> +++ b/dotx.d/layouterror_res.x
> @@ -0,0 +1,5 @@
> +
> +struct LAYOUTERROR4res {
> +	nfsstat4	ler_status;
> +};
> +
> diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
> index 4a6d254..980ad23 100755
> --- a/dotx.d/spit_types.sh
> +++ b/dotx.d/spit_types.sh
> @@ -470,21 +470,6 @@ union layoutreturn4 switch(layoutreturn_type4 lr_returntype) {
>  EOF
>          ;;
>  
> -	type_layoutreturn_errs.x )
> -
> -cat << EOF > $i
> -struct layoutreturn_device_error4 {
> -	deviceid4	lrde_deviceid;
> -	nfsstat4	lrde_status;
> -	nfs_opnum4	lrde_opnum;
> -};
> -
> -struct layoutreturn_error_report4 {
> -	layoutreturn_device_error4	lrer_errors<>;
> -};
> -EOF
> -        ;;
> -
>  	type_client_owner4.x )
>  
>  cat << EOF > $i
> diff --git a/dotx.d/type_device_error.x b/dotx.d/type_device_error.x
> new file mode 100644
> index 0000000..8fc6733
> --- /dev/null
> +++ b/dotx.d/type_device_error.x
> @@ -0,0 +1,7 @@
> +
> +struct device_error4 {
> +	deviceid4	de_deviceid;

Tom, the server may store multiple data files on the same deviceid4
so something like the data file filehandle is required to identify to particular
container.

One use case for example is a data file that hit a media failure and needs
reconstruction from a replica.

Benny

> +	nfsstat4	de_status;
> +	nfs_opnum4	de_opnum;
> +};
> +
> diff --git a/nfsv42_middle_errors.xml b/nfsv42_middle_errors.xml
> index 73f255c..ef7a5f9 100644
> --- a/nfsv42_middle_errors.xml
> +++ b/nfsv42_middle_errors.xml
> @@ -1885,6 +1885,37 @@
>          NFS4ERR_WRONG_CRED
>        </c>
>  
> +      <c>LAYOUTERROR</c>
> +      <c>
> +        NFS4ERR_ADMIN_REVOKED,
> +        NFS4ERR_BADXDR,
> +        NFS4ERR_BAD_STATEID,
> +        NFS4ERR_DEADSESSION,
> +        NFS4ERR_DELAY,
> +        NFS4ERR_DELEG_REVOKED,
> +        NFS4ERR_EXPIRED,
> +        NFS4ERR_FHEXPIRED,
> +        NFS4ERR_GRACE,
> +        NFS4ERR_INVAL,
> +        NFS4ERR_ISDIR,
> +        NFS4ERR_MOVED,
> +        NFS4ERR_NOFILEHANDLE,
> +        NFS4ERR_NOTSUPP,
> +        NFS4ERR_NO_GRACE,
> +        NFS4ERR_OLD_STATEID,
> +        NFS4ERR_OP_NOT_IN_SESSION,
> +        NFS4ERR_REP_TOO_BIG,
> +        NFS4ERR_REP_TOO_BIG_TO_CACHE,
> +        NFS4ERR_REQ_TOO_BIG,
> +        NFS4ERR_RETRY_UNCACHED_REP,
> +        NFS4ERR_SERVERFAULT,
> +        NFS4ERR_STALE,
> +        NFS4ERR_TOO_MANY_OPS,
> +        NFS4ERR_UNKNOWN_LAYOUTTYPE,
> +        NFS4ERR_WRONG_CRED,
> +        NFS4ERR_WRONG_TYPE
> +      </c>
> +
>        <c>LAYOUTGET</c>
>        <c>
>          NFS4ERR_ACCESS,
> diff --git a/nfsv42_middle_new_errors.xml b/nfsv42_middle_new_errors.xml
> index cc18168..17d31a4 100644
> --- a/nfsv42_middle_new_errors.xml
> +++ b/nfsv42_middle_new_errors.xml
> @@ -269,6 +269,37 @@
>          NFS4ERR_WRONG_TYPE
>        </c>
>  
> +      <c>LAYOUTERROR</c>
> +      <c>
> +        NFS4ERR_ADMIN_REVOKED,
> +        NFS4ERR_BADXDR,
> +        NFS4ERR_BAD_STATEID,
> +        NFS4ERR_DEADSESSION,
> +        NFS4ERR_DELAY,
> +        NFS4ERR_DELEG_REVOKED,
> +        NFS4ERR_EXPIRED,
> +        NFS4ERR_FHEXPIRED,
> +        NFS4ERR_GRACE,
> +        NFS4ERR_INVAL,
> +        NFS4ERR_ISDIR,
> +        NFS4ERR_MOVED,
> +        NFS4ERR_NOFILEHANDLE,
> +        NFS4ERR_NOTSUPP,
> +        NFS4ERR_NO_GRACE,
> +        NFS4ERR_OLD_STATEID,
> +        NFS4ERR_OP_NOT_IN_SESSION,
> +        NFS4ERR_REP_TOO_BIG,
> +        NFS4ERR_REP_TOO_BIG_TO_CACHE,
> +        NFS4ERR_REQ_TOO_BIG,
> +        NFS4ERR_RETRY_UNCACHED_REP,
> +        NFS4ERR_SERVERFAULT,
> +        NFS4ERR_STALE,
> +        NFS4ERR_TOO_MANY_OPS,
> +        NFS4ERR_UNKNOWN_LAYOUTTYPE,
> +        NFS4ERR_WRONG_CRED,
> +        NFS4ERR_WRONG_TYPE
> +      </c>
> +
>        <c>LAYOUTSTATS</c>
>        <c>
>          NFS4ERR_ADMIN_REVOKED,
> diff --git a/nfsv42_middle_op_layouterror.xml b/nfsv42_middle_op_layouterror.xml
> new file mode 100644
> index 0000000..e51d46d
> --- /dev/null
> +++ b/nfsv42_middle_op_layouterror.xml
> @@ -0,0 +1,147 @@
> +<!-- Copyright (C) The IETF Trust (2011-2014) -->
> +<!-- Copyright (C) The Internet Society (2011-2014) -->
> +
> +<section anchor='op:layouterror' title='Operation XX: LAYOUTERROR - Provide Errors for the Layout'>
> +  <section toc='exclude' title="ARGUMENT">
> +    <?rfc include='autogen/type_layoutupdate4.xml'?>
> +    <?rfc include='autogen/type_device_error.xml'?>
> +    <?rfc include='autogen/layouterror_args.xml'?>
> +  </section>
> +
> +  <section toc='exclude' title="RESULT">
> +    <?rfc include='autogen/layouterror_res.xml'?>
> +  </section>
> +
> +  <section toc='exclude' title="DESCRIPTION">
> +    <t>
> +      The client can use LAYOUTERROR to inform the metadata server
> +      about errors in its interaction with the layout represented by the
> +      current filehandle, client ID (derived from the session ID
> +      in the preceding SEQUENCE operation), byte-range (lea_offset
> +      + lea_length), and lea_stateid.
> +    </t>
> +
> +    <t>
> +      Each individual device_error4 describes a single error
> +      associated with a storage device, which is identified via
> +      de_deviceid.  The operation which returned the error is
> +      identified via de_opnum. Finally the NFS error value (nfsstat4)
> +      encountered is provided via de_status and may consist of the
> +      following error codes:
> +    </t>
> +
> +    <t>
> +      <list style='hanging'>
> +        <t hangText='NFS4ERR_NXIO:'>
> +          The client was unable to establish any communication with
> +          the storage device.
> +        </t>
> +
> +        <t hangText='NFS4ERR_*:'>
> +          The client was able to establish communication with the
> +          storage device and is returning one of the allowed error
> +          codes for the operation denoted by de_opnum.
> +        </t>
> +      </list>
> +    </t>
> +  </section>
> +
> +  <section anchor="sec:LAYOUTERROR:impl" toc='exclude' title="IMPLEMENTATION">
> +    <t>
> +      There are two broad classes of errors, transient and persistent.
> +      The client SHOULD strive to only use this new mechanism to
> +      report persistent errors. It MUST be able to deal with transient
> +      issues by itself. Also, while the client might consider an
> +      issue to be persistent, it MUST be prepared for the metadata
> +      server to consider such issues to be transient. A prime example
> +      of this is if the metadata server fences off a client from
> +      either a stateid or a filehandle.  The client will get an
> +      error from the storage device and might relay either
> +      NFS4ERR_ACCESS or NFS4ERR_BAD_STATEID back to the metadata
> +      server, with the belief that this is a hard error. If the
> +      metadata server is informed by the client that there is an
> +      error, it can safely ignore that.  For it, the mission is
> +      accomplished in that the client has returned a layout that
> +      the metadata server had most likely recalled.
> +    </t>
> +
> +    <t>
> +      The client might also need to inform the metadata server that
> +      it cannot reach one or more of the storage devices. While the
> +      metadata server can detect the connectivity of both of these
> +      paths:
> +
> +      <list style='symbols'>
> +        <t>
> +          metadata server to storage device
> +        </t>
> +
> +        <t>
> +          metadata server to client
> +        </t>
> +      </list>
> +
> +      it cannot determine if the client and storage device path is
> +      working. As with the case of the storage device passing errors
> +      to the client, it must be prepared for the metadata server
> +      to consider such outages as being transitory.
> +    </t>
> +
> +    <t>
> +      Clients are expected to tolerate transient storage device
> +      errors, and hence clients SHOULD NOT use the LAYOUTERROR
> +      error handling for device access problems that may be transient.
> +      The methods by which a client decides whether a device access
> +      problem is transient vs persistent are implementation-specific,
> +      but may include retrying I/Os to a data server under appropriate
> +      conditions.
> +    </t>
> +
> +    <t>
> +      When an I/O fails to a storage device, the client SHOULD retry
> +      the failed I/O via the metadata server. In this situation,
> +      before retrying the I/O, the client SHOULD return the layout, or
> +      the affected portion thereof, and SHOULD indicate which storage
> +      device or devices was problematic. The client needs to do this
> +      when the storage device is being unresponsive in order to fence
> +      off any failed write attempts, and ensure that they do not end
> +      up overwriting any later data being written through the metadata
> +      server.  If the client does not do this, the metadata server MAY
> +      issue a layout recall callback in order to perform the retried I/O.
> +    </t>
> +
> +    <t>
> +      The client needs to be cognizant that since this error handling
> +      is optional in the metadata server, the metadata server may
> +      silently ignore this functionality.  Also, as the metadata
> +      server may consider some issues the client reports to be
> +      expected, the client might find it difficult to detect a
> +      metadata server which has not implemented error handling via
> +      LAYOUTERROR.
> +    </t>
> +
> +    <t>
> +      If an metadata server is aware that a storage device is proving
> +      problematic to a client, the metadata server SHOULD NOT include
> +      that storage device in any pNFS layouts sent to that client. If
> +      the metadata server is aware that a storage device is affecting
> +      many clients, then the metadata server SHOULD NOT include that
> +      storage device in any pNFS layouts sent out.  If a client asks
> +      for a new layout for the file from the metadata server, it MUST
> +      be prepared for the metadata server to return that storage device
> +      in the layout. The metadata server might not have any choice in
> +      using the storage device, i.e., there might only be one possible
> +      layout for the system.  Also, in the case of existing files, the
> +      metadata server might have no choice in which storage devices to
> +      hand out to clients.
> +    </t>
> +
> +    <t>
> +      The metadata server is not required to indefinitely retain
> +      per-client storage device error information. An metadata server is
> +      also not required to automatically reinstate use of a previously
> +      problematic storage device; administrative intervention may be
> +      required instead.
> +    </t>
> +  </section>
> +</section>
> diff --git a/nfsv42_middle_op_layoutreturn.xml b/nfsv42_middle_op_layoutreturn.xml
> deleted file mode 100644
> index ad2f33d..0000000
> --- a/nfsv42_middle_op_layoutreturn.xml
> +++ /dev/null
> @@ -1,185 +0,0 @@
> -<!-- Copyright (C) The IETF Trust (2011-2014) -->
> -<!-- Copyright (C) The Internet Society (2011-2014) -->
> -
> -<section anchor="sec:pLeh:LAYOUTRETURN" title="Changes to Operation 51: LAYOUTRETURN">
> -  <section anchor="sec:pLeh:intro" toc='exclude' title="Introduction">
> -    <t>
> -      In the pNFS description provided in <xref target="RFC5661" />, the
> -      client is not capable of relaying an error code from the storage
> -      device to the metadata server. In the specification of the Object
> -      Layout Type <xref target='RFC5664' />, use is made of the opaque
> -      lrf_body field of the LAYOUTRETURN argument to do such a relaying
> -      of error codes. In this section, we define a new data structure
> -      to enable the passing of error codes back to the metadata server
> -      and provide some guidelines on what both the client and metadata
> -      server should expect in such circumstances.
> -    </t>
> -
> -    <t>
> -      There are two broad classes of errors, transient and persistent.
> -      The client SHOULD strive to only use this new mechanism to report
> -      persistent errors. It MUST be able to deal with transient issues
> -      by itself. Also, while the client might consider an issue to
> -      be persistent, it MUST be prepared for the metadata server to
> -      consider such issues to be transient. A prime example of this is
> -      if the metadata server fences off a client from either a stateid
> -      or a filehandle.  The client will get an error from the storage
> -      device and might relay either NFS4ERR_ACCESS or NFS4ERR_BAD_STATEID
> -      back to the metadata server, with the belief that this is a hard
> -      error. If the metadata server is informed by the client that there
> -      is an error, it can safely ignore that.  For it, the mission is
> -      accomplished in that the client has returned a layout that the
> -      metadata server had most likely recalled.
> -    </t>
> -
> -    <t>
> -      The client might also need to inform the metadata server that
> -      it cannot reach one or more of the storage devices. While the
> -      metadata server can detect the connectivity of both of these paths:
> -
> -      <list style='symbols'>
> -        <t>
> -          metadata server to storage device
> -        </t>
> -
> -        <t>
> -          metadata server to client
> -        </t>
> -      </list>
> -
> -      it cannot determine if the client and storage device path is
> -      working. As with the case of the storage device passing errors to
> -      the client, it must be prepared for the metadata server to consider
> -      such outages as being transitory.
> -    </t>
> -
> -    <t>
> -      The existing LAYOUTRETURN operation is extended by introducing a
> -      new data structure to report errors, layoutreturn_device_error4.
> -      Also, layoutreturn_device_error4 is introduced to enable an
> -      array of errors to be reported.
> -    </t>
> -  </section>
> -
> -  <section anchor="sec:pLeh:LAYOUTRETURN:args" toc='exclude' title="ARGUMENT">
> -    <t>
> -      The ARGUMENT specification of the LAYOUTRETURN operation in section
> -      18.44.1 of <xref target="RFC5661" /> is augmented by the following
> -      XDR code <xref target="RFC4506" />:
> -    </t>
> -
> -    <?rfc include='autogen/type_layoutreturn_errs.xml'?>
> -  </section>
> -
> -  <section anchor="sec:pLeh:LAYOUTRETURN:res" toc='exclude' title="RESULT">
> -    <t>
> -      The RESULT of the LAYOUTRETURN operation is unchanged; see section
> -      18.44.2 of <xref target="RFC5661" />.
> -    </t>
> -  </section>
> -
> -  <section anchor="sec:pLeh:LAYOUTRETURN:desc" toc='exclude' title="DESCRIPTION">
> -    <t>
> -      The following text is added to the end of the LAYOUTRETURN operation
> -      DESCRIPTION in section 18.44.3 of <xref target="RFC5661" />.
> -    </t>
> -
> -    <t>
> -      When a client uses LAYOUTRETURN with a type of LAYOUTRETURN4_FILE,
> -      then if the lrf_body field is NULL, it indicates to the metadata
> -      server that the client experienced no errors. If lrf_body is
> -      non-NULL, then the field references error information which is
> -      layout type specific. I.e., the Object Layout Type can continue to
> -      utilize lrf_body as specified in <xref target='RFC5664' />.  For
> -      both File <xref target="RFC5661" /> and Block <xref target="RFC5663"
> -      /> Layout Types, the field references a layoutreturn_device_error4,
> -      which contains an array of layoutreturn_device_error4.
> -    </t>
> -
> -    <t>
> -      Each individual layoutreturn_device_error4 describes a single
> -      error associated with a storage device, which is identified
> -      via lrde_deviceid.  The operation which returned the error is
> -      identified via lrde_opnum. Finally the NFS error value (nfsstat4)
> -      encountered is provided via lrde_status and may consist of the
> -      following error codes:
> -    </t>
> -
> -    <t>
> -      <list style='hanging'>
> -        <t hangText='NFS4ERR_NXIO:'>
> -          The client was unable to establish any communication with the storage device.
> -        </t>
> -
> -        <t hangText='NFS4ERR_*:'>
> -          The client was able to establish communication with the storage device and
> -          is returning one of the allowed error codes for the operation
> -          denoted by lrde_opnum.
> -        </t>
> -      </list>
> -    </t>
> -  </section>
> -
> -  <section anchor="sec:pLeh:LAYOUTRETURN:impl" toc='exclude' title="IMPLEMENTATION">
> -    <t>
> -      The following text is added to the end of the LAYOUTRETURN operation
> -      IMPLEMENTATION in section 18.4.4 of <xref target="RFC5661" />.
> -    </t>
> -
> -    <t>
> -      Clients are expected to tolerate transient storage device errors,
> -      and hence clients SHOULD NOT use the LAYOUTRETURN error handling
> -      for device access problems that may be transient. The methods by
> -      which a client decides whether a device access problem is transient
> -      vs persistent are implementation-specific, but may include retrying
> -      I/Os to a data server under appropriate conditions.
> -    </t>
> -
> -    <t>
> -      When an I/O fails to a storage device, the client SHOULD retry
> -      the failed I/O via the metadata server. In this situation,
> -      before retrying the I/O, the client SHOULD return the layout, or
> -      the affected portion thereof, and SHOULD indicate which storage
> -      device or devices was problematic. The client needs to do this
> -      when the storage device is being unresponsive in order to fence
> -      off any failed write attempts, and ensure that they do not end
> -      up overwriting any later data being written through the metadata
> -      server.  If the client does not do this, the metadata server MAY
> -      issue a layout recall callback in order to perform the retried I/O.
> -    </t>
> -
> -    <t>
> -      The client needs to be cognizant that since this error handling is
> -      optional in the metadata server, the metadata server may silently
> -      ignore this functionality.  Also, as the metadata server may
> -      consider some issues the client reports to be expected (see <xref
> -      target='sec:pLeh:intro' />), the client might find it difficult to
> -      detect a metadata server which has not implemented error handling
> -      via LAYOUTRETURN.
> -    </t>
> -
> -    <t>
> -      If an metadata server is aware that a storage device is proving
> -      problematic to a client, the metadata server SHOULD NOT include
> -      that storage device in any pNFS layouts sent to that client. If
> -      the metadata server is aware that a storage device is affecting
> -      many clients, then the metadata server SHOULD NOT include that
> -      storage device in any pNFS layouts sent out.  If a client asks
> -      for a new layout for the file from the metadata server, it MUST
> -      be prepared for the metadata server to return that storage device
> -      in the layout. The metadata server might not have any choice in
> -      using the storage device, i.e., there might only be one possible
> -      layout for the system.  Also, in the case of existing files, the
> -      metadata server might have no choice in which storage devices to
> -      hand out to clients.
> -    </t>
> -
> -    <t>
> -      The metadata server is not required to indefinitely retain
> -      per-client storage device error information. An metadata server is
> -      also not required to automatically reinstate use of a previously
> -      problematic storage device; administrative intervention may be
> -      required instead.
> -    </t>
> -  </section>
> -</section>
> diff --git a/nfsv42_middle_op_layoutstats.xml b/nfsv42_middle_op_layoutstats.xml
> index 4784b13..93f3747 100644
> --- a/nfsv42_middle_op_layoutstats.xml
> +++ b/nfsv42_middle_op_layoutstats.xml
> @@ -1,7 +1,7 @@
>  <!-- Copyright (C) The IETF Trust (2014) -->
>  <!-- Copyright (C) The Internet Society (2014) -->
>  
> -<section anchor='op:layout_event' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
> +<section anchor='op:layoutstats' title='Operation 70: LAYOUTSTATS - Provide Statistics for the Layout'>
>    <section toc='exclude' title="ARGUMENT">
>      <?rfc include='autogen/type_layoutupdate4.xml'?>
>      <?rfc include='autogen/type_io_info.xml'?>
> diff --git a/nfsv42_middle_op_mandlist.xml b/nfsv42_middle_op_mandlist.xml
> index b590a6e..8664f06 100644
> --- a/nfsv42_middle_op_mandlist.xml
> +++ b/nfsv42_middle_op_mandlist.xml
> @@ -141,7 +141,8 @@
>        <c>LAYOUTCOMMIT         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
>        <c>LAYOUTGET            </c> <c> OPT </c> <c> pNFS (REQ)   </c>
>        <c>LAYOUTRETURN         </c> <c> OPT </c> <c> pNFS (REQ)   </c>
> -      <c>LAYOUTSTATS          </c> <c> OPT </c> <c> pNFS (REQ)   </c>
> +      <c>LAYOUTERROR          </c> <c> OPT </c> <c> pNFS (OPT)   </c>
> +      <c>LAYOUTSTATS          </c> <c> OPT </c> <c> pNFS (OPT)   </c>
>        <c>LINK                 </c> <c> OPT </c> <c>              </c>
>        <c>LOCK                 </c> <c> REQ </c> <c>              </c>
>        <c>LOCKT                </c> <c> REQ </c> <c>              </c>