@@ -333,14 +333,14 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in
333333 return rm ;
334334}
335335
336- int rds_message_copy_from_user (struct rds_message * rm , struct iov_iter * from ,
337- bool zcopy )
336+ int rds_message_zcopy_from_user (struct rds_message * rm , struct iov_iter * from )
338337{
339- unsigned long to_copy , nbytes ;
340338 unsigned long sg_off ;
341339 struct scatterlist * sg ;
342340 int ret = 0 ;
343341 int length = iov_iter_count (from );
342+ int total_copied = 0 ;
343+ struct sk_buff * skb ;
344344
345345 rm -> m_inc .i_hdr .h_len = cpu_to_be32 (iov_iter_count (from ));
346346
@@ -350,54 +350,66 @@ int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
350350 sg = rm -> data .op_sg ;
351351 sg_off = 0 ; /* Dear gcc, sg->page will be null from kzalloc. */
352352
353- if (zcopy ) {
354- int total_copied = 0 ;
355- struct sk_buff * skb ;
356-
357- skb = alloc_skb (0 , GFP_KERNEL );
358- if (!skb )
359- return - ENOMEM ;
360- BUILD_BUG_ON (sizeof (skb -> cb ) <
361- max_t (int , sizeof (struct rds_znotifier ),
362- sizeof (struct rds_zcopy_cookies )));
363- rm -> data .op_mmp_znotifier = RDS_ZCOPY_SKB (skb );
364- if (mm_account_pinned_pages (& rm -> data .op_mmp_znotifier -> z_mmp ,
365- length )) {
366- ret = - ENOMEM ;
353+ skb = alloc_skb (0 , GFP_KERNEL );
354+ if (!skb )
355+ return - ENOMEM ;
356+ BUILD_BUG_ON (sizeof (skb -> cb ) < max_t (int , sizeof (struct rds_znotifier ),
357+ sizeof (struct rds_zcopy_cookies )));
358+ rm -> data .op_mmp_znotifier = RDS_ZCOPY_SKB (skb );
359+ if (mm_account_pinned_pages (& rm -> data .op_mmp_znotifier -> z_mmp ,
360+ length )) {
361+ ret = - ENOMEM ;
362+ goto err ;
363+ }
364+ while (iov_iter_count (from )) {
365+ struct page * pages ;
366+ size_t start ;
367+ ssize_t copied ;
368+
369+ copied = iov_iter_get_pages (from , & pages , PAGE_SIZE ,
370+ 1 , & start );
371+ if (copied < 0 ) {
372+ struct mmpin * mmp ;
373+ int i ;
374+
375+ for (i = 0 ; i < rm -> data .op_nents ; i ++ )
376+ put_page (sg_page (& rm -> data .op_sg [i ]));
377+ mmp = & rm -> data .op_mmp_znotifier -> z_mmp ;
378+ mm_unaccount_pinned_pages (mmp );
379+ ret = - EFAULT ;
367380 goto err ;
368381 }
369- while (iov_iter_count (from )) {
370- struct page * pages ;
371- size_t start ;
372- ssize_t copied ;
373-
374- copied = iov_iter_get_pages (from , & pages , PAGE_SIZE ,
375- 1 , & start );
376- if (copied < 0 ) {
377- struct mmpin * mmp ;
378- int i ;
379-
380- for (i = 0 ; i < rm -> data .op_nents ; i ++ )
381- put_page (sg_page (& rm -> data .op_sg [i ]));
382- mmp = & rm -> data .op_mmp_znotifier -> z_mmp ;
383- mm_unaccount_pinned_pages (mmp );
384- ret = - EFAULT ;
385- goto err ;
386- }
387- total_copied += copied ;
388- iov_iter_advance (from , copied );
389- length -= copied ;
390- sg_set_page (sg , pages , copied , start );
391- rm -> data .op_nents ++ ;
392- sg ++ ;
393- }
394- WARN_ON_ONCE (length != 0 );
395- return ret ;
382+ total_copied += copied ;
383+ iov_iter_advance (from , copied );
384+ length -= copied ;
385+ sg_set_page (sg , pages , copied , start );
386+ rm -> data .op_nents ++ ;
387+ sg ++ ;
388+ }
389+ WARN_ON_ONCE (length != 0 );
390+ return ret ;
396391err :
397- consume_skb (skb );
398- rm -> data .op_mmp_znotifier = NULL ;
399- return ret ;
400- } /* zcopy */
392+ consume_skb (skb );
393+ rm -> data .op_mmp_znotifier = NULL ;
394+ return ret ;
395+ }
396+
397+ int rds_message_copy_from_user (struct rds_message * rm , struct iov_iter * from ,
398+ bool zcopy )
399+ {
400+ unsigned long to_copy , nbytes ;
401+ unsigned long sg_off ;
402+ struct scatterlist * sg ;
403+ int ret = 0 ;
404+
405+ rm -> m_inc .i_hdr .h_len = cpu_to_be32 (iov_iter_count (from ));
406+
407+ /* now allocate and copy in the data payload. */
408+ sg = rm -> data .op_sg ;
409+ sg_off = 0 ; /* Dear gcc, sg->page will be null from kzalloc. */
410+
411+ if (zcopy )
412+ return rds_message_zcopy_from_user (rm , from );
401413
402414 while (iov_iter_count (from )) {
403415 if (!sg_page (sg )) {
0 commit comments